From d4c278e0d1de42ae9344cc7af8035b2dec7cdec7 Mon Sep 17 00:00:00 2001 From: eleith Date: Thu, 12 Nov 2026 07:35:46 -0800 Subject: [PATCH] feat(reports): prepare shared native transaction exports --- .../prepare-shared-native-transactions.md | 5 + .../prepare-reports/run.integration.test.ts | 71 ++++++- .../src/workflows/prepare-reports/run.test.ts | 13 +- .../src/workflows/prepare-reports/run.ts | 5 +- .../src/accounts/balances.integration.test.ts | 6 +- packages/hledger/src/index.ts | 1 + .../hledger/src/overview/net-worth.test.ts | 36 +++- packages/hledger/src/overview/ranges.test.ts | 6 +- packages/hledger/src/overview/report.test.ts | 27 ++- .../transaction-counts.integration.test.ts | 6 +- .../src/prepare-reports/account-stream.ts | 48 +++-- .../hledger/src/prepare-reports/batch.test.ts | 37 ++++ packages/hledger/src/prepare-reports/batch.ts | 55 ++--- .../prepare-reports/run.integration.test.ts | 32 ++- .../hledger/src/prepare-reports/run.test.ts | 181 +++++++++++------ packages/hledger/src/prepare-reports/run.ts | 16 +- .../prepare-reports/transactions.fixture.ts | 55 +++++ packages/hledger/src/report-stream.test.ts | 12 +- packages/hledger/src/report-stream.ts | 12 +- .../src/transactions/account-rows.test.ts | 48 +++-- .../hledger/src/transactions/export.test.ts | 110 ++-------- packages/hledger/src/transactions/export.ts | 69 +------ packages/hledger/src/transactions/index.ts | 1 + .../src/transactions/investment.test.ts | 190 +++++++++--------- .../hledger/src/transactions/investment.ts | 8 +- packages/hledger/src/transactions/store.ts | 13 ++ .../snapshots/src/reports/publication.test.ts | 91 +++++++++ packages/snapshots/src/reports/publication.ts | 50 ++++- 28 files changed, 772 insertions(+), 432 deletions(-) create mode 100644 .changeset/prepare-shared-native-transactions.md create mode 100644 packages/hledger/src/prepare-reports/batch.test.ts create mode 100644 packages/hledger/src/prepare-reports/transactions.fixture.ts create mode 100644 packages/hledger/src/transactions/store.ts diff --git a/.changeset/prepare-shared-native-transactions.md b/.changeset/prepare-shared-native-transactions.md new file mode 100644 index 0000000..2077838 --- /dev/null +++ b/.changeset/prepare-shared-native-transactions.md @@ -0,0 +1,5 @@ +--- +'@hledger-reports/worker': patch +--- + +Prepare one shared pair of native transaction exports instead of repeating them for every account. diff --git a/apps/worker/src/workflows/prepare-reports/run.integration.test.ts b/apps/worker/src/workflows/prepare-reports/run.integration.test.ts index 01bb95f..7526103 100644 --- a/apps/worker/src/workflows/prepare-reports/run.integration.test.ts +++ b/apps/worker/src/workflows/prepare-reports/run.integration.test.ts @@ -2,14 +2,17 @@ import { enqueuePreparation } from './test-fixtures.js'; import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { afterEach, expect, it } from 'vitest'; import { openJobsClient, prepareReportsSpec, type JobsClient } from '@hledger-reports/jobs'; +import { decodeNativeTransaction } from '@hledger-reports/hledger/transactions'; import { createReportStage, readAccountPage, readOverview, readReportPublication, - readValidation + readValidation, + type ReportPublication } from '@hledger-reports/snapshots'; import { lotsJournal as source } from '../../../fixtures/journal.js'; import { startWorker } from '../../index.js'; @@ -59,6 +62,71 @@ async function settled(client: JobsClient, status: 'completed' | 'failed' = 'com return (await preparations(client))[0]!; } +function expectSavedNativeExports( + publication: ReportPublication, + accountRowCount: number, + includedSource: string +) { + const database = new DatabaseSync(join(publication.directory, 'reports.sqlite'), { + readOnly: true + }); + try { + const stored = database + .prepare( + 'SELECT native_index, original_json, transacted_json FROM transactions ORDER BY native_index' + ) + .all(); + expect(stored).toHaveLength(publication.overview.journalCoverage.transactionCount); + expect(stored.length).toBeGreaterThan(accountRowCount); + const pairs = stored.map((record) => { + expect(record.original_json).toEqual(expect.any(String)); + expect(record.transacted_json).toEqual(expect.any(String)); + const original = decodeNativeTransaction(record.original_json as string); + const transacted = decodeNativeTransaction(record.transacted_json as string); + for (const transaction of [original, transacted]) { + expect(transaction.tindex).toBe(record.native_index); + expect(transaction.tpostings.length).toBeGreaterThan(0); + expect(transaction.tsourcepos).toContainEqual( + expect.objectContaining({ sourceName: includedSource, sourceLine: expect.any(Number) }) + ); + } + expect(transacted.tdescription).toBe(original.tdescription); + return { original, transacted }; + }); + const purchase = pairs.find( + ({ original }) => original.tdescription === 'Brokerage | Buy VTI and BND' + )!; + expect(purchase.original.tpostings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + paccount: 'assets:investments:brokerage', + pamount: expect.arrayContaining([ + expect.objectContaining({ + acommodity: 'VTI', + acost: expect.objectContaining({ + tag: 'UnitCost', + contents: expect.objectContaining({ acommodity: 'USD' }) + }) + }) + ]) + }), + expect.objectContaining({ paccount: 'assets:bank:checking' }) + ]) + ); + const transfer = pairs.find( + ({ original }) => original.tdescription === 'Transfer | Move two VTI to retirement' + )!; + for (const transaction of [transfer.original, transfer.transacted]) { + expect(transaction.tpostings.map((posting) => posting.paccount)).toEqual([ + 'assets:investments:brokerage', + 'assets:investments:retirement' + ]); + } + } finally { + database.close(); + } +} + it('explicitly prepares a saved overview through the real SDK without a browser and reloads with the worker stopped', async () => { const f = await fixture(); const client = await openJobsClient(f.jobs); @@ -122,6 +190,7 @@ it('explicitly prepares a saved overview through the real SDK without a browser limit: 50 }); expect(savedPage).toEqual(page); + expectSavedNativeExports(savedPublication, page.rows.length, f.included); expect(await readFile(f.journal, 'utf8')).toBe('include included.journal\n'); expect(await readFile(f.included, 'utf8')).toBe(source); } finally { diff --git a/apps/worker/src/workflows/prepare-reports/run.test.ts b/apps/worker/src/workflows/prepare-reports/run.test.ts index 30e39d1..addfb10 100644 --- a/apps/worker/src/workflows/prepare-reports/run.test.ts +++ b/apps/worker/src/workflows/prepare-reports/run.test.ts @@ -9,7 +9,8 @@ import { createReportStage, readReportPublication, writeValidation, - type ReportPublication + type ReportPublication, + type ReportStage } from '@hledger-reports/snapshots'; import { createPrepareReports } from './run.js'; @@ -50,10 +51,17 @@ const previous: ReportPublication = { directory: '/owned/snapshots/.sets/fixture' }; const stage = { + transactions: { + insertOriginal: vi.fn(), + attachTransacted: vi.fn(), + assertComplete: vi.fn(), + read: vi.fn(), + accountHistory: vi.fn() + }, writeRows: vi.fn(async () => {}), publish: vi.fn(async () => {}), discard: vi.fn(async () => {}) -}; +} satisfies ReportStage; const prepare = createPrepareReports(journal, directory); const run = { id: 'example', createdAt: new Date('2026-04-16T00:00:00Z') }; const checkpoint = vi.fn(); @@ -142,6 +150,7 @@ it.each([true, false])( vi.mocked(createReportStage).mock.invocationCallOrder[0]! ); expect(prepareReports).toHaveBeenCalledExactlyOnceWith(journal, { + transactions: stage.transactions, writeRows: stage.writeRows }); expect(stage.writeRows).toHaveBeenCalledExactlyOnceWith('assets:cash', []); diff --git a/apps/worker/src/workflows/prepare-reports/run.ts b/apps/worker/src/workflows/prepare-reports/run.ts index 8ee21c4..8780a7d 100644 --- a/apps/worker/src/workflows/prepare-reports/run.ts +++ b/apps/worker/src/workflows/prepare-reports/run.ts @@ -32,7 +32,10 @@ export function createPrepareReports(journal: string, directory: string): Prepar previous?.journal === journal ? previous.accounts : null ); try { - const reports = await prepareReports(journal, { writeRows: stage.writeRows }); + const reports = await prepareReports(journal, { + transactions: stage.transactions, + writeRows: stage.writeRows + }); await stage.publish(reports.overview, reports.accounts, journal); return reports.overview; } finally { diff --git a/packages/hledger/src/accounts/balances.integration.test.ts b/packages/hledger/src/accounts/balances.integration.test.ts index 86c318f..b53da93 100644 --- a/packages/hledger/src/accounts/balances.integration.test.ts +++ b/packages/hledger/src/accounts/balances.integration.test.ts @@ -7,6 +7,7 @@ import { expect, it } from 'vitest'; import { lotsJournal } from '../../../../apps/worker/fixtures/journal.js'; import { formatQuantity, parseExactJson, type Amount } from '../amounts.js'; import { prepareReports } from '../prepare-reports/run.js'; +import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; const exec = promisify(execFile); @@ -18,7 +19,10 @@ const amounts = (bag: Amount[]) => })); async function compareNativeSubtree(journal: string, name: string) { - const prepared = await prepareReports(journal, { async writeRows() {} }); + const prepared = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + async writeRows() {} + }); const { asOf } = prepared.overview; const base = prepared.overview.netWorth.currency!; expect(asOf).not.toBeNull(); diff --git a/packages/hledger/src/index.ts b/packages/hledger/src/index.ts index 5186a73..ea5595c 100644 --- a/packages/hledger/src/index.ts +++ b/packages/hledger/src/index.ts @@ -16,6 +16,7 @@ export type { ReportedAmount } from './amounts.js'; export { runHoldings } from './holdings/report.js'; export type { HoldingPosition, HoldingsAccount, HoldingsReport } from './holdings/report.js'; export { prepareReports } from './prepare-reports/run.js'; +export type { PreparationTransactions } from './transactions/store.js'; export type { PreparedAccountSummary } from './prepare-reports/batch.js'; export type { PreparedBalance } from './accounts/balances.js'; export type { PreparedAccountRow } from './transactions/account-rows.js'; diff --git a/packages/hledger/src/overview/net-worth.test.ts b/packages/hledger/src/overview/net-worth.test.ts index b6eb1a4..cd9fc37 100644 --- a/packages/hledger/src/overview/net-worth.test.ts +++ b/packages/hledger/src/overview/net-worth.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { prepareReports, getHledgerVersion } from '../index.js'; +import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; import { balanceFrames, decodeOverviewFixture, @@ -212,7 +213,10 @@ describe('native market assets minus liabilities snapshots', () => { txn('2026-03-01', 'Debt', 'Wallet', '100.00 USD') + txn('2026-04-16', 'Costs', 'Wallet', '1000.00 USD') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth).toEqual({ currency: 'USD', points: [ @@ -236,7 +240,10 @@ describe('native market assets minus liabilities snapshots', () => { journal, 'commodity USD 1.00\n' + accounts + txn('2026-04-16', 'Wallet', 'Debt', '100.00 USD') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth?.points.slice(0, 11)).toEqual( dates.slice(0, 11).map((date) => ({ date, amounts: [], unconverted: [] })) ); @@ -256,7 +263,10 @@ describe('native market assets minus liabilities snapshots', () => { txn('2025-01-01', 'Shares', 'Equity', '2 STOCK') + txn('2026-04-16', 'Wallet', 'Equity', '1.00 USD') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth).toEqual({ currency: 'USD', points: [...Array(6).fill('40'), '40', '40', '80', '80', '80', '121.00'].map( @@ -276,7 +286,10 @@ describe('native market assets minus liabilities snapshots', () => { accounts + 'P 2025-01-01 USD 1.00 USD\n\n2026-04-16 Purchase\n Shares 2 STOCK @ 50.00 USD\n Wallet -100.00 USD\n' ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth?.currency).toBe('USD'); expect(data.netWorth?.points[11]).toEqual({ date: dates[11], @@ -293,7 +306,10 @@ describe('native market assets minus liabilities snapshots', () => { 'P 2025-01-01 STOCK 10.00 EUR\nP 2099-01-01 EUR 2.00 USD\nP 2099-02-01 EUR 3.00 USD\n' + txn('2026-04-16', 'Shares', 'Equity', '2 STOCK') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth?.currency).toBe('USD'); expect(data.netWorth?.points[11]).toEqual({ date: dates[11], @@ -310,7 +326,10 @@ describe('native market assets minus liabilities snapshots', () => { 'P 2025-01-01 USD 1.00 USD\n' + txn('2026-04-16', 'Shares', 'Debt', '2 STOCK') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth?.currency).toBe('USD'); expect(data.netWorth?.points[11]?.unconverted).toEqual(['STOCK']); expect(data.netWorth?.points[11]?.amounts).toEqual([q('0', 'STOCK')]); @@ -330,7 +349,10 @@ describe('native market assets minus liabilities snapshots', () => { accounts + txn('2026-04-16', 'Wallet', 'Equity', `42.00 ${symbol}`) ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.netWorth?.currency).toBe(currency); expect(data.netWorth?.points[11]).toEqual({ date: dates[11], diff --git a/packages/hledger/src/overview/ranges.test.ts b/packages/hledger/src/overview/ranges.test.ts index 6c7c860..7838539 100644 --- a/packages/hledger/src/overview/ranges.test.ts +++ b/packages/hledger/src/overview/ranges.test.ts @@ -6,6 +6,7 @@ import { planIncomeRanges, organizeIncomeRange } from './ranges.js'; import type { HledgerIncomeStatement } from './income-statement.js'; import type { Amount } from '../amounts.js'; import { prepareReports, getHledgerVersion } from '../index.js'; +import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; const plan = planIncomeRanges('2026-04-16')[3]; function statement(range = plan): HledgerIncomeStatement { @@ -162,7 +163,10 @@ describe('native hledger 1.99.5 finite overview ranges', () => { '\n2026-04-12 Later posting\n Costs:Food 700.00 USD ; date:2026-04-17\n Wallet\n' + '\n~ monthly from 2025-05-01\n Costs:Food 900.00 USD\n Wallet\n'; await writeFile(journal, text); - const { overview } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(overview.asOf).toBe('2026-04-16'); for (const [months, begin, income, expenses, net, food] of [ [1, '2026-04-01', '400.00', '35.00', '365.00', '40.00'], diff --git a/packages/hledger/src/overview/report.test.ts b/packages/hledger/src/overview/report.test.ts index c1ad7b5..75b5e87 100644 --- a/packages/hledger/src/overview/report.test.ts +++ b/packages/hledger/src/overview/report.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { prepareReports, getHledgerVersion } from '../index.js'; +import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; import { balanceFrames, decodeOverviewFixture, @@ -126,7 +127,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { it('keeps twelve empty income columns and four empty ranges with a leap-day asset anchor', async () => { await writeFile(journal, declarations + posting('2024-02-29', 'Wallet', 'Reserve', '1.00 USD')); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.asOf).toBe('2024-02-29'); expect(data.incomeExpenses.months).toHaveLength(12); for (const month of data.incomeExpenses.months) { @@ -149,7 +153,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { declarations + 'P 2099-12-31 EUR 2.00 USD\n\n~ monthly from 2099-01-01\n Spending 1.00 USD\n Wallet\n' ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data).toEqual({ asOf: null, journalCoverage: emptyCoverage, @@ -174,7 +181,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { '\n2040-06-01 Earlier posting date\n Wallet 2.00 USD\n Paycheck -2.00 USD ; date:2039-01-01\n' + '\n~ monthly from 2099-01-01\n Spending 1.00 USD\n Wallet\n' ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.asOf).toBe('2040-07-12'); expect(data.journalCoverage).toEqual({ transactionCount: 5, @@ -195,7 +205,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { posting('2026-04-02', 'Spending', 'Wallet', '100.00 USD') + posting('2026-05-03', 'Wallet', 'Investment gains', '250.00 USD') ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.asOf).toBe('2026-05-03'); expect(data.incomeExpenses.months.find(({ begin }) => begin === '2026-04-01')).toMatchObject({ income: [{ commodity: 'USD', quantity: '1000.00' }], @@ -216,7 +229,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { '\n2026-04-02 Stock expense\n Spending 2 STOCK @ 50.00 USD\n Wallet -100.00 USD\n' + '\n2026-04-03 Other stock cost\n Spending 3 STOCK @ 60.00 USD\n Wallet -180.00 USD\n' ); - const { overview: data } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview: data } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect(data.asOf).toBe('2026-04-03'); const bag = [ { commodity: 'EUR', quantity: '9007199254740993.06' }, @@ -238,6 +254,7 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => { declarations + posting('2026-04-15', 'SECRET undeclared', 'Wallet', '123.45 USD') ); const error: unknown = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), writeRows: async () => {} }).catch((error: unknown) => error); expect(error).toBeInstanceOf(Error); diff --git a/packages/hledger/src/overview/transaction-counts.integration.test.ts b/packages/hledger/src/overview/transaction-counts.integration.test.ts index 565a7b1..3fc66cb 100644 --- a/packages/hledger/src/overview/transaction-counts.integration.test.ts +++ b/packages/hledger/src/overview/transaction-counts.integration.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { afterAll, beforeAll, expect, it } from 'vitest'; import { transactionCountsJournal } from '../../fixtures/transaction-counts.fixture.js'; import { prepareReports } from '../prepare-reports/run.js'; +import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; let directory: string; let journal: string; @@ -18,7 +19,10 @@ afterAll(async () => { const expected = { 1: 2, 3: 4, 6: 6, 12: 8 }; it('prepares all four native counts alongside whole coverage and the existing financial set', async () => { - const { overview } = await prepareReports(journal, { writeRows: async () => {} }); + const { overview } = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + writeRows: async () => {} + }); expect( Object.fromEntries( Object.entries(overview.ranges!).map(([months, range]) => [months, range.transactionCount]) diff --git a/packages/hledger/src/prepare-reports/account-stream.ts b/packages/hledger/src/prepare-reports/account-stream.ts index 59fb6be..15c5780 100644 --- a/packages/hledger/src/prepare-reports/account-stream.ts +++ b/packages/hledger/src/prepare-reports/account-stream.ts @@ -4,28 +4,47 @@ import { organizeAccountRow, type PreparedAccountRow } from '../transactions/account-rows.js'; -import { AccountTransactionExports } from '../transactions/export.js'; +import { decodeNativeTransaction } from '../transactions/export.js'; import { accountInvestmentContext, type InvestmentMetadata } from '../transactions/investment.js'; -import { accountMarkers, type PreparedAccountSummary } from './batch.js'; +import type { NativeTransactionPair, PreparationTransactions } from '../transactions/store.js'; +import { prepareTransaction, requireTransactionIndex } from '../transactions/transaction.js'; +import type { PreparedAccountSummary } from './batch.js'; export type AccountRowWriter = ( accountName: string, rows: readonly PreparedAccountRow[] ) => Promise; +function prepareAccountTransaction(pair: NativeTransactionPair, accountName: string) { + const original = decodeNativeTransaction(pair.originalJson); + const transaction = prepareTransaction(original, decodeNativeTransaction(pair.transactedJson)); + if ( + !transaction.postings.some( + (posting) => posting.account === accountName || posting.account.startsWith(`${accountName}:`) + ) + ) { + throw new Error('Native transaction does not belong to the account subtree.'); + } + return transaction; +} + export async function writePreparedAccountRows( reader: ReportStreamReader, account: PreparedAccountSummary, metadata: InvestmentMetadata, + transactions: PreparationTransactions, writeRows: AccountRowWriter, check: () => void ): Promise { - const exports = new AccountTransactionExports(account.name, account.access === 'ambiguous'); - await reader.array('object', async (json) => exports.original(json)); - await reader.expect(`\n${accountMarkers.originalEnd}`); - await reader.array('object', async (json) => exports.transacted(json)); - await reader.expect(`\n${accountMarkers.transactedEnd}`); - const context = accountInvestmentContext(exports.transactions(), metadata, account.name); + if (account.access === 'ambiguous') return; + async function* history() { + for await (const pair of transactions.accountHistory(account.name)) { + check(); + yield prepareAccountTransaction(pair, account.name); + } + } + const context = await accountInvestmentContext(history(), metadata, account.name); + const indexes = new Set(); let chunk: PreparedAccountRow[] = []; const write = async () => { check(); @@ -36,13 +55,18 @@ export async function writePreparedAccountRows( chunk = []; }; await reader.array('array', async (json) => { + check(); const tuple = parseAccountRegisterRow(json); - if (account.access === 'ambiguous') return; - const transaction = exports.take(tuple[0].tindex); + const index = tuple[0].tindex; + requireTransactionIndex(index); + if (indexes.has(index)) throw new Error('Missing or duplicate native register index.'); + indexes.add(index); + const pair = await transactions.read(index); + check(); + const transaction = prepareAccountTransaction(pair, account.name); chunk.push(organizeAccountRow(tuple, account.name, transaction, context)); account.rowCount = (account.rowCount ?? 0) + 1; if (chunk.length === 50) await write(); }); - await reader.expect(`\n${accountMarkers.registerEnd}`); - if (account.access === 'ready' && (chunk.length > 0 || account.rowCount === 0)) await write(); + if (chunk.length > 0 || account.rowCount === 0) await write(); } diff --git a/packages/hledger/src/prepare-reports/batch.test.ts b/packages/hledger/src/prepare-reports/batch.test.ts new file mode 100644 index 0000000..ddf8db3 --- /dev/null +++ b/packages/hledger/src/prepare-reports/batch.test.ts @@ -0,0 +1,37 @@ +import { expect, it } from 'vitest'; +import { journalMetadata } from '../journal-stats.js'; +import { planPreparedReportCommands } from './batch.js'; + +it('exports the whole journal once regardless of account count or ambiguous selections', () => { + const { args } = planPreparedReportCommands([], { + journal: journalMetadata('Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'), + accounts: [ + { name: 'assets:cash', access: 'ready' }, + { name: 'assets:cash:daily', access: 'ready' }, + { name: 'assets:Case', access: 'ambiguous' } + ], + investment: { + assets: new Set(), + cash: new Set(), + liabilities: new Set(), + income: new Set(), + expenses: new Set(), + gains: new Set(), + lotCommodities: new Set() + } + }); + expect(args.filter((argument) => argument === 'print')).toHaveLength(2); + expect(args.slice(args.indexOf('print'), args.indexOf('aregister'))).toEqual([ + 'print', + '--round=none', + '-Ojson', + '--', + 'print', + '--round=none', + '--value=transacted', + '-Ojson', + '--' + ]); + expect(args.filter((argument) => argument === 'aregister')).toHaveLength(2); + expect(args).not.toContain('^assets:Case$'); +}); diff --git a/packages/hledger/src/prepare-reports/batch.ts b/packages/hledger/src/prepare-reports/batch.ts index 7e1a078..4f00e23 100644 --- a/packages/hledger/src/prepare-reports/batch.ts +++ b/packages/hledger/src/prepare-reports/batch.ts @@ -13,12 +13,8 @@ export interface PreparedAccountSummary extends QualifiedAccount { summary: PreparedBalance; } -export const registersMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n'; -export const accountMarkers = { - originalEnd: '\n__HLEDGER_PREPARED_ORIGINAL_END__\n', - transactedEnd: '\n__HLEDGER_PREPARED_TRANSACTED_END__\n', - registerEnd: '\n__HLEDGER_PREPARED_REGISTER_END__\n' -} as const; +// Keep the legacy delimiter for the finite financial prefix. +export const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n'; function echo(marker: string): string[] { return ['--', 'echo', marker.trimEnd()]; @@ -39,41 +35,22 @@ export function planPreparedReportCommands( '--', ...overviewPlan.commands, ...planAccountBalances(metadata.journal), - ...echo(registersMarker) + ...echo(financialEndMarker), + '--', + 'print', + '--round=none', + '-Ojson', + '--', + 'print', + '--round=none', + '--value=transacted', + '-Ojson' ]; - for (const { name } of metadata.accounts) { - const escapedName = escapeAccountName(name); - const exactAccountPattern = `^${escapedName}$`; - // Print's acct query is case-insensitive; exact canonical membership is - // checked after decoding, without dropping any counterpart postings. - const subtreeQuery = `acct:^${escapedName}(:|$)`; - args.push( - '--', - 'print', - '--round=none', - '-Ojson', - subtreeQuery, - ...echo(accountMarkers.originalEnd) - ); - args.push( - '--', - 'print', - '--round=none', - '--value=transacted', - '-Ojson', - subtreeQuery, - ...echo(accountMarkers.transactedEnd) - ); + for (const { name, access } of metadata.accounts) { + if (access === 'ambiguous') continue; + const exactAccountPattern = `^${escapeAccountName(name)}$`; // No separator before the literal pattern: run owns `--` separators. - args.push( - '--', - 'aregister', - '--historical', - '--empty', - '-Ojson', - exactAccountPattern, - ...echo(accountMarkers.registerEnd) - ); + args.push('--', 'aregister', '--historical', '--empty', '-Ojson', exactAccountPattern); } return { args, overviewPlan }; } diff --git a/packages/hledger/src/prepare-reports/run.integration.test.ts b/packages/hledger/src/prepare-reports/run.integration.test.ts index 4363f8a..d7769a8 100644 --- a/packages/hledger/src/prepare-reports/run.integration.test.ts +++ b/packages/hledger/src/prepare-reports/run.integration.test.ts @@ -4,7 +4,10 @@ import { join } from 'node:path'; import { afterEach, expect, it } from 'vitest'; import { cashJournal } from '../../fixtures/cash.fixture.js'; import type { PreparedAccountRow } from '../transactions/account-rows.js'; +import { decodeNativeTransaction } from '../transactions/export.js'; +import { prepareTransaction } from '../transactions/transaction.js'; import { prepareReports } from './run.js'; +import { MemoryPreparationTransactions } from './transactions.fixture.js'; const directories: string[] = []; @@ -26,6 +29,7 @@ function collectRows() { const rowsByAccount = new Map(); return { rowsByAccount, + transactions: new MemoryPreparationTransactions(), async writeRows(name: string, rows: readonly PreparedAccountRow[]) { const previous = rowsByAccount.get(name) ?? []; previous.push(...rows); @@ -94,6 +98,7 @@ include detail.journal const during = collectRows(); let changed = false; const reports = await prepareReports(journal, { + transactions: during.transactions, async writeRows(name, rows) { await during.writeRows(name, rows); if (!changed) { @@ -108,7 +113,10 @@ include detail.journal ...beforeReports.overview, preparedAt: '' }); - const after = await prepareReports(journal, { async writeRows() {} }); + const after = await prepareReports(journal, { + transactions: new MemoryPreparationTransactions(), + async writeRows() {} + }); expect(after.overview.netWorth).not.toEqual(reports.overview.netWorth); }); @@ -121,6 +129,10 @@ account equity assets:case 1 USD assets:Case 2 USD equity + +2026-01-02 Only ambiguous accounts + assets:case 3 USD + assets:Case -3 USD `); const collected = collectRows(); const prepared = await prepareReports(journal, collected); @@ -130,4 +142,22 @@ account equity expect(account).toMatchObject({ access: 'ambiguous', rowCount: null }); expect(collected.rowsByAccount.has(account.name)).toBe(false); } + expect(collected.transactions.records.size).toBe(2); + const pairs = await Promise.all( + [...collected.transactions.records.keys()].map((index) => collected.transactions.read(index)) + ); + const transactions = pairs.map(({ originalJson, transactedJson }) => + prepareTransaction( + decodeNativeTransaction(originalJson), + decodeNativeTransaction(transactedJson) + ) + ); + expect(transactions.map(({ description }) => description)).toEqual([ + 'Collision', + 'Only ambiguous accounts' + ]); + expect(transactions[1].postings).toMatchObject([ + { account: 'assets:case', transactedAmounts: [{ commodity: 'USD', quantity: '3' }] }, + { account: 'assets:Case', transactedAmounts: [{ commodity: 'USD', quantity: '-3' }] } + ]); }); diff --git a/packages/hledger/src/prepare-reports/run.test.ts b/packages/hledger/src/prepare-reports/run.test.ts index 3d357d5..2ab5ad4 100644 --- a/packages/hledger/src/prepare-reports/run.test.ts +++ b/packages/hledger/src/prepare-reports/run.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { supportedHledgerVersion } from '../checks.js'; import { transactionFixture } from '../transactions/transaction.fixture.js'; import { prepareReports } from './run.js'; +import { MemoryPreparationTransactions } from './transactions.fixture.js'; const processMock = vi.hoisted(() => ({ spawn: vi.fn() })); vi.mock('node:child_process', async (importOriginal) => { @@ -35,16 +36,14 @@ afterEach(async () => { const versionOutput = `hledger ${supportedHledgerVersion}, SECRET banner\n`; const journalStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'; -const financialPrefix = `${journalStats}\n__HLEDGER_PREPARED_REGISTERS__\n`; -const originalEnd = '\n\n__HLEDGER_PREPARED_ORIGINAL_END__\n'; -const transactedEnd = '\n\n__HLEDGER_PREPARED_TRANSACTED_END__\n'; -const registerEnd = '\n\n__HLEDGER_PREPARED_REGISTER_END__\n'; +const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n'; +const financialPrefix = `${journalStats}${financialEndMarker}`; const nativeTransaction = { ...transactionFixture(), tdate: '2031-01-01', tdescription: 'λ, test' }; -const accountExports = `${financialPrefix}[${JSON.stringify(nativeTransaction)}]${originalEnd}[${JSON.stringify(nativeTransaction)}]${transactedEnd}`; +const globalExports = `${financialPrefix}[${JSON.stringify(nativeTransaction)}]\n[${JSON.stringify(nativeTransaction)}]\n`; const registerRow = JSON.stringify([ nativeTransaction, nativeTransaction, @@ -67,7 +66,7 @@ function accountProtocol(rowCount: number): string { const rows = JSON.stringify( transactions.map((transaction) => [transaction, transaction, false, ['other, 東京'], [], []]) ); - return `${financialPrefix}${prints}${originalEnd}${prints}${transactedEnd}${rows}${registerEnd}`; + return `${financialPrefix}${prints}\n${prints}\n${rows}`; } function startFixtureChild(script: string): ChildProcessWithoutNullStreams { @@ -114,9 +113,10 @@ function mockPreparationProcesses( async function expectRejected(output: string, exitCode = 0) { const child = mockPreparationProcesses(output, { exitCode }); const writeRows = vi.fn(async () => {}); - const error = await prepareReports('/private/SECRET', { writeRows }).catch( - (error: unknown) => error - ); + const error = await prepareReports('/private/SECRET', { + transactions: new MemoryPreparationTransactions(), + writeRows + }).catch((error: unknown) => error); expect(error).toBeInstanceOf(Error); expect((error as Error).message).not.toMatch(/SECRET|\/private/); expect((error as Error).cause).toBeUndefined(); @@ -129,8 +129,9 @@ describe('preparation completion and failure', () => { vi.useFakeTimers({ toFake: ['Date'] }); try { vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); - mockPreparationProcesses(`${accountExports}[${registerRow}]${registerEnd}`); + mockPreparationProcesses(`${globalExports}[${registerRow}] \t\r\n`); const result = await prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), async writeRows() { vi.setSystemTime(new Date('2026-01-01T00:02:30Z')); } @@ -152,18 +153,48 @@ describe('preparation completion and failure', () => { }); it.each([ - { name: 'truncated account output', output: `${accountExports}[${registerRow}` }, + { name: 'truncated account output', output: `${globalExports}[${registerRow}` }, { - name: 'crossed export frames', - output: `${accountExports}[${registerRow}]${registerEnd}`.replace( - '__HLEDGER_PREPARED_ORIGINAL_END__', - '__WRONG_END__' - ) + name: 'malformed native JSON between global exports', + output: `${financialPrefix}[${JSON.stringify(nativeTransaction)}]invalid[]` } ])('rejects $name without returning a partial report', async ({ output }) => { await expectRejected(output); }); + it('rejects a repeated register transaction even though shared storage permits rereads', async () => { + const writes = await expectRejected(`${globalExports}[${registerRow},${registerRow}]`); + expect(writes).not.toHaveBeenCalled(); + }); + + it('rejects incomplete global pairs even when every account is ambiguous', async () => { + queueChildOutput(versionOutput); + queueChildOutput(journalStats); + queueChildOutput('assets:case\nassets:Case\n'); + queueChildOutput('assets:case ; type: C\nassets:Case ; type: C\n'); + queueChildOutput(''); + queueChildOutput('assets:case\nassets:case\n'); + const original = { + ...nativeTransaction, + tpostings: nativeTransaction.tpostings.map((posting, index) => ({ + ...posting, + paccount: index === 0 ? 'assets:case' : 'assets:Case' + })) + }; + queueChildOutput(`${financialPrefix}[${JSON.stringify(original)}]\n[]\n`); + const transactions = new MemoryPreparationTransactions(); + const assertComplete = vi.spyOn(transactions, 'assertComplete'); + const writeRows = vi.fn(async () => {}); + await expect( + prepareReports('/private', { + transactions, + writeRows + }) + ).rejects.toThrow('Prepared transactions could not be read. Check the journal.'); + expect(assertComplete).toHaveBeenCalledOnce(); + expect(writeRows).not.toHaveBeenCalled(); + }); + it('rejects a late native nonzero exit even after row writes', async () => { const writes = await expectRejected(accountProtocol(51), 9); expect(writes).toHaveBeenCalled(); @@ -172,6 +203,7 @@ describe('preparation completion and failure', () => { it('kills the child and withholds private diagnostics when a row callback fails', async () => { const child = mockPreparationProcesses(accountProtocol(51), { keepOpen: true }); const error = await prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), async writeRows() { throw new Error('SECRET /private 123.45 USD'); } @@ -182,57 +214,73 @@ describe('preparation completion and failure', () => { expect(child.signalCode).toBe('SIGKILL'); }); - it('kills the child on abort but waits for the active row write before returning', async () => { - const child = mockPreparationProcesses(accountProtocol(50), { keepOpen: true }); - const childClosed = new Promise((resolve) => { - child.once('close', () => resolve()); - }); - const controller = new AbortController(); - let enterWrite!: () => void; - const firstWrite = new Promise((resolve) => { - enterWrite = resolve; - }); - let releaseWrite!: () => void; - const held = new Promise((resolve) => { - releaseWrite = resolve; - }); - releaseWrites.push(releaseWrite); - let writeFinished = false; - let settled = false; - const result = prepareReports('/private', { - signal: controller.signal, - async writeRows() { + it.each(['original import', 'account rows'] as const)( + 'kills the child on abort but waits for the active %s write', + async (phase) => { + const child = mockPreparationProcesses(accountProtocol(50), { keepOpen: true }); + const childClosed = new Promise((resolve) => { + child.once('close', () => resolve()); + }); + const controller = new AbortController(); + let enterWrite!: () => void; + const firstWrite = new Promise((resolve) => { + enterWrite = resolve; + }); + let releaseWrite!: () => void; + const held = new Promise((resolve) => { + releaseWrite = resolve; + }); + releaseWrites.push(releaseWrite); + let writeFinished = false; + let settled = false; + const transactions = new MemoryPreparationTransactions(); + const write = async () => { enterWrite(); await held; writeFinished = true; + }; + if (phase === 'original import') { + const insertOriginal = transactions.insertOriginal.bind(transactions); + vi.spyOn(transactions, 'insertOriginal').mockImplementationOnce(async (json) => { + await write(); + await insertOriginal(json); + }); } - }); - void result.then( - () => { - settled = true; - }, - () => { - settled = true; + const result = prepareReports('/private', { + transactions, + signal: controller.signal, + async writeRows() { + if (phase === 'account rows') await write(); + } + }); + void result.then( + () => { + settled = true; + }, + () => { + settled = true; + } + ); + try { + await firstWrite; + controller.abort(); + await childClosed; + expect(child.signalCode).toBe('SIGKILL'); + expect(settled).toBe(false); + expect(writeFinished).toBe(false); + } finally { + releaseWrite(); } - ); - try { - await firstWrite; - controller.abort(); - await childClosed; - expect(child.signalCode).toBe('SIGKILL'); - expect(settled).toBe(false); - expect(writeFinished).toBe(false); - } finally { - releaseWrite(); + await expect(result).rejects.toThrow(); + expect(writeFinished).toBe(true); } - await expect(result).rejects.toThrow(); - expect(writeFinished).toBe(true); - }); + ); it('kills a blocked financial read when aborted', async () => { - const child = mockPreparationProcesses(accountExports.slice(0, -5), { keepOpen: true }); + const child = mockPreparationProcesses(globalExports.slice(0, -5), { keepOpen: true }); const controller = new AbortController(); const result = prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), signal: controller.signal, async writeRows() {} }); @@ -252,6 +300,7 @@ describe('preparation completion and failure', () => { }); const controller = new AbortController(); const result = prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), signal: controller.signal, async writeRows() {} }); @@ -267,25 +316,27 @@ describe('preparation completion and failure', () => { describe('financial framing and metadata qualification', () => { it('does not confuse financial stats-path markers with protocol boundaries', async () => { mockPreparationProcesses( - `Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${accountExports}[]${registerEnd}` + `Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${globalExports}[]` ); - const result = await prepareReports('/private', { async writeRows() {} }); + const result = await prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), + async writeRows() {} + }); expect(result.accounts[0].rowCount).toBe(0); }); it('rejects a forged complete protocol in a stats path unless physical EOF also validates', async () => { - await expectRejected( - `${accountExports}[]${registerEnd}rest-of-path\n${accountExports}[]${registerEnd}` - ); + await expectRejected(`${globalExports}[]rest-of-path\n${globalExports}[]`); }); it('rejects unsafe account identities before starting financial execution', async () => { queueChildOutput(versionOutput); queueChildOutput(journalStats); queueChildOutput('assets:\tSECRET\n'); - const error = await prepareReports('/private/SECRET', { async writeRows() {} }).catch( - (error: unknown) => error - ); + const error = await prepareReports('/private/SECRET', { + transactions: new MemoryPreparationTransactions(), + async writeRows() {} + }).catch((error: unknown) => error); expect(error).toBeInstanceOf(Error); expect((error as Error).message).not.toMatch(/SECRET|\/private/); expect((error as Error).cause).toBeUndefined(); diff --git a/packages/hledger/src/prepare-reports/run.ts b/packages/hledger/src/prepare-reports/run.ts index f69042b..329bb56 100644 --- a/packages/hledger/src/prepare-reports/run.ts +++ b/packages/hledger/src/prepare-reports/run.ts @@ -3,17 +3,19 @@ import type { OverviewData } from '../overview/report.js'; import { consumeHledger, readHledger } from '../process.js'; import { ReportStreamReader } from '../report-stream.js'; import { readPreparationMetadata } from './metadata.js'; +import type { PreparationTransactions } from '../transactions/store.js'; import { writePreparedAccountRows, type AccountRowWriter } from './account-stream.js'; import { decodePreparedPrefix, planPreparedReportCommands, - registersMarker, + financialEndMarker, type PreparedAccountSummary } from './batch.js'; export async function prepareReports( journal: string, options: { + transactions: PreparationTransactions; writeRows: AccountRowWriter; signal?: AbortSignal; } @@ -31,14 +33,24 @@ export async function prepareReports( { signal: options.signal }, async (output, check) => { const reader = new ReportStreamReader(output, check); - const { overview, accounts } = await reader.prefix(registersMarker, (prefix) => + const { overview, accounts } = await reader.prefix(financialEndMarker, (prefix) => decodePreparedPrefix(prefix, metadata, overviewPlan) ); + await reader.array('object', async (json) => { + await options.transactions.insertOriginal(json); + }); + await reader.array('object', async (json) => { + await options.transactions.attachTransacted(json); + }); + check(); + await options.transactions.assertComplete(); + check(); for (const account of accounts) { await writePreparedAccountRows( reader, account, metadata.investment, + options.transactions, options.writeRows, check ); diff --git a/packages/hledger/src/prepare-reports/transactions.fixture.ts b/packages/hledger/src/prepare-reports/transactions.fixture.ts new file mode 100644 index 0000000..7f00556 --- /dev/null +++ b/packages/hledger/src/prepare-reports/transactions.fixture.ts @@ -0,0 +1,55 @@ +import { decodeNativeTransaction } from '../transactions/export.js'; +import type { NativeTransactionPair, PreparationTransactions } from '../transactions/store.js'; +import { + assertTransactionExportsMatch, + requireTransactionIndex +} from '../transactions/transaction.js'; + +/** Owned, in-memory storage for SDK preparation tests only. */ +export class MemoryPreparationTransactions implements PreparationTransactions { + readonly records = new Map(); + + async insertOriginal(json: string): Promise { + const transaction = decodeNativeTransaction(json); + if (this.records.has(transaction.tindex)) throw new Error('Duplicate native original index.'); + this.records.set(transaction.tindex, { originalJson: json, transactedJson: null }); + } + + async attachTransacted(json: string): Promise { + const transaction = decodeNativeTransaction(json); + const record = this.records.get(transaction.tindex); + if (!record) throw new Error('Missing native original index.'); + if (record.transactedJson !== null) throw new Error('Duplicate native transacted index.'); + assertTransactionExportsMatch(decodeNativeTransaction(record.originalJson), transaction); + record.transactedJson = json; + } + + async assertComplete(): Promise { + for (const record of this.records.values()) { + if (record.transactedJson === null) throw new Error('Missing native transacted index.'); + } + } + + async read(index: number): Promise { + requireTransactionIndex(index); + const record = this.records.get(index); + if (!record || record.transactedJson === null) + throw new Error('Missing native transaction pair.'); + return { originalJson: record.originalJson, transactedJson: record.transactedJson }; + } + + async *accountHistory(accountName: string): AsyncIterable { + for (const index of [...this.records.keys()].sort((left, right) => left - right)) { + const pair = await this.read(index); + const transaction = decodeNativeTransaction(pair.originalJson); + if ( + transaction.tpostings.some(({ paccount }) => { + if (typeof paccount !== 'string') throw new Error('Invalid native posting account.'); + return paccount === accountName || paccount.startsWith(`${accountName}:`); + }) + ) { + yield pair; + } + } + } +} diff --git a/packages/hledger/src/report-stream.test.ts b/packages/hledger/src/report-stream.test.ts index 33aecb5..b2613b3 100644 --- a/packages/hledger/src/report-stream.test.ts +++ b/packages/hledger/src/report-stream.test.ts @@ -13,19 +13,23 @@ function reader(text: string | Buffer, split = 1) { } it.each(['array', 'object'] as const)( - 'frames %s records separately across UTF-8 and escaped string boundaries', + 'reads consecutive native %s arrays across UTF-8 and escaped strings', async (kind) => { const value = kind === 'object' ? { text: '東京 λ \\" ] }', nested: [{ key: [1, 2] }] } : [{ text: '東京 λ \\" ] }' }]; - const input = reader(JSON.stringify([value, value]) + '\nEND'); + const input = reader( + JSON.stringify([value, value]) + '\n' + JSON.stringify([value]) + ' \t\r\n' + ); const rows: unknown[] = []; await input.array(kind, async (json) => { rows.push(JSON.parse(json)); }); - expect(rows).toEqual([value, value]); - await input.expect('\nEND'); + await input.array(kind, async (json) => { + rows.push(JSON.parse(json)); + }); + expect(rows).toEqual([value, value, value]); await input.eof(); } ); diff --git a/packages/hledger/src/report-stream.ts b/packages/hledger/src/report-stream.ts index aed98ee..732af45 100644 --- a/packages/hledger/src/report-stream.ts +++ b/packages/hledger/src/report-stream.ts @@ -56,16 +56,10 @@ export class ReportStreamReader { throw new Error('Missing native prefix.'); } - async expect(literal: string): Promise { - for (const char of literal) { - if (!(await this.more()) || this.text[this.offset++] !== char) { - throw new Error('Invalid native separator.'); - } - } - } - async eof(): Promise { - if (await this.more()) throw new Error('Extra native output.'); + while (await this.more()) { + if (!/^[ \t\r\n]$/.test(this.text[this.offset++])) throw new Error('Extra native output.'); + } } async array(kind: 'array' | 'object', row: (json: string) => Promise): Promise { diff --git a/packages/hledger/src/transactions/account-rows.test.ts b/packages/hledger/src/transactions/account-rows.test.ts index 05b43b0..0103f70 100644 --- a/packages/hledger/src/transactions/account-rows.test.ts +++ b/packages/hledger/src/transactions/account-rows.test.ts @@ -13,13 +13,13 @@ const metadata: InvestmentMetadata = { gains: new Set(), lotCommodities: new Set() }; -function organize(input: ReturnType, account: string) { +async function organize(input: ReturnType, account: string) { const transaction = prepareTransaction(input[0], structuredClone(input[0])); return organizeAccountRow( input, account, transaction, - accountInvestmentContext([transaction], metadata, account) + await accountInvestmentContext([transaction], metadata, account) ); } @@ -37,11 +37,11 @@ function tuple() { ); } -it('retains original metadata alongside account-context date and exact reported bags', () => { +it('retains original metadata alongside account-context date and exact reported bags', async () => { const input = tuple(); input[4][0].aquantity.decimalMantissa = -900719925474099301n; const before = structuredClone(input); - const result = organize(input, 'assets:cash'); + const result = await organize(input, 'assets:cash'); expect(result).toMatchObject({ accountDate: '2026-04-16', isSplit: true, @@ -61,7 +61,7 @@ it('retains original metadata alongside account-context date and exact reported expect(JSON.parse(JSON.stringify(result))).toEqual(result); }); -it('matches qualified canonical account/subtree case-sensitively, independent of OtherAccounts', () => { +it('matches qualified canonical account/subtree case-sensitively, independent of OtherAccounts', async () => { const original = transactionFixture(); const names = [ 'assets:cash', @@ -79,10 +79,10 @@ it('matches qualified canonical account/subtree case-sensitively, independent of const input = parseAccountRegisterRow( JSON.stringify([original, original, false, ['assets:cash', 'assets:cash:Child'], [], []]) ); - expect(organize(input, 'assets:cash').accountPostingIndexes).toEqual([0, 1, 5]); + expect((await organize(input, 'assets:cash')).accountPostingIndexes).toEqual([0, 1, 5]); }); -it('preserves ordered duplicate commodities, mixed units, unnamed and explicit zero entries', () => { +it('preserves ordered duplicate commodities, mixed units, unnamed and explicit zero entries', async () => { const input = tuple(); input[4] = [ { @@ -96,7 +96,7 @@ it('preserves ordered duplicate commodities, mixed units, unnamed and explicit z input[5] = [ { acommodity: '', aquantity: { decimalMantissa: 0n, decimalPlaces: 2 }, acost: null } ]; - const result = organize(input, 'assets:cash'); + const result = await organize(input, 'assets:cash'); expect(result.change).toEqual([ { commodity: 'USD', quantity: '9007199254740993.01', cost: null, costBasis: null, style: null }, { commodity: 'USD', quantity: '-0.0001', cost: null, costBasis: null, style: null }, @@ -111,19 +111,23 @@ it.each(['{}', '[]', '[{}, {}, false, [], []]'])('rejects an incomplete tuple: % expect(() => parseAccountRegisterRow(json)).toThrow('Prepared transactions could not be read.'); }); -it('rejects unexpected contextual header differences rather than losing information', () => { +it('rejects unexpected contextual header differences rather than losing information', async () => { const input = tuple(); input[1].tdate2 = 'unexpected'; - expect(() => organize(input, 'assets:cash')).toThrow('Prepared transactions could not be read.'); + await expect(organize(input, 'assets:cash')).rejects.toThrow( + 'Prepared transactions could not be read.' + ); }); -it('rejects contextual posting changes', () => { +it('rejects contextual posting changes', async () => { const input = tuple(); input[1].tpostings.pop(); - expect(() => organize(input, 'assets:cash')).toThrow('Prepared transactions could not be read.'); + await expect(organize(input, 'assets:cash')).rejects.toThrow( + 'Prepared transactions could not be read.' + ); }); -it('recomputes selected indexes from original print, not stripped register posting positions', () => { +it('recomputes selected indexes from original print, not stripped register posting positions', async () => { const input = tuple(); const print = structuredClone(input[0]); print.tpostings.unshift({ ...print.tpostings[1], paccount: 'equity:conversion' }); @@ -132,7 +136,7 @@ it('recomputes selected indexes from original print, not stripped register posti input, 'assets:cash', transaction, - accountInvestmentContext([transaction], metadata, 'assets:cash') + await accountInvestmentContext([transaction], metadata, 'assets:cash') ); expect(result.accountPostingIndexes).toEqual([1]); expect(result.transaction).toBe(transaction); @@ -141,20 +145,14 @@ it('recomputes selected indexes from original print, not stripped register posti it.each(['tindex', 'ttags'] as const)( 'rejects a joined print whose stable %s identity differs', - (field) => { + async (field) => { const input = tuple(); const print = structuredClone(input[0]); if (field === 'tindex') print[field]++; else print[field].push(['different', '']); const transaction = prepareTransaction(print, structuredClone(print)); - expect(() => - organizeAccountRow( - input, - 'assets:cash', - transaction, - accountInvestmentContext([transaction], metadata, 'assets:cash') - ) - ).toThrow('header'); + const context = await accountInvestmentContext([transaction], metadata, 'assets:cash'); + expect(() => organizeAccountRow(input, 'assets:cash', transaction, context)).toThrow('header'); } ); @@ -163,7 +161,7 @@ it.each([ ['counterpart', 'memo:outside', false] ] as const)( 'uses account-local historical virtual evidence (%s), not arbitrary counterpart virtual postings', - (_kind, virtualAccount, unclassified) => { + async (_kind, virtualAccount, unclassified) => { const input = tuple(); input[5] = structuredClone(input[4]); const transaction = prepareTransaction(input[0], structuredClone(input[0])); @@ -172,7 +170,7 @@ it.each([ earlier.tpostings[1].paccount = virtualAccount; earlier.tpostings[1].preal = 'VirtualPosting'; const history = prepareTransaction(earlier, structuredClone(earlier)); - const context = accountInvestmentContext([history, transaction], metadata, 'assets:cash'); + const context = await accountInvestmentContext([history, transaction], metadata, 'assets:cash'); const row = organizeAccountRow(input, 'assets:cash', transaction, context); const amounts = [{ commodity: 'USD', quantity: '-100.00' }]; expect(row.investment.cash.change).toEqual(unclassified ? [] : amounts); diff --git a/packages/hledger/src/transactions/export.test.ts b/packages/hledger/src/transactions/export.test.ts index 62a92de..cd57d88 100644 --- a/packages/hledger/src/transactions/export.test.ts +++ b/packages/hledger/src/transactions/export.test.ts @@ -1,98 +1,26 @@ import { expect, it } from 'vitest'; -import { AccountTransactionExports } from './export.js'; +import { decodeNativeTransaction } from './export.js'; import { transactionFixture } from './transaction.fixture.js'; -function native(index = 7, account = 'assets:cash') { - const transaction = transactionFixture(); - transaction.tindex = index; - transaction.tpostings[0].paccount = account; - return transaction; -} -const json = (value: unknown) => JSON.stringify(value); - -it('joins reordered exports by positive native index, not equal dates/descriptions/amounts', () => { - const exports = new AccountTransactionExports('assets:cash', false); - for (const index of [7, 9, 8]) exports.original(json(native(index))); - for (const index of [8, 7, 9]) exports.transacted(json(native(index))); - expect([...exports.transactions()].map((t) => t.index)).toEqual([7, 9, 8]); - expect([9, 8, 7].map((index) => exports.take(index).index)).toEqual([9, 8, 7]); - expect(() => exports.take(7)).toThrow('Missing or duplicate'); -}); - -it('narrows CI print selection by exact canonical/subtree membership while retaining all counterparts', () => { - const exports = new AccountTransactionExports('assets:Cash', false); - const selected = native(1, 'assets:Cash:東京'); - selected.tpostings.push({ ...selected.tpostings[1], paccount: 'assets:cash:other' }); - const extras = [native(2, 'assets:cash'), native(3, 'assets:Cashbox')]; - for (const transaction of [selected, ...extras]) exports.original(json(transaction)); - for (const transaction of [...extras, selected]) exports.transacted(json(transaction)); - expect([...exports.transactions()]).toHaveLength(1); - expect(exports.take(1).postings.map((p) => p.account)).toEqual([ - 'assets:Cash:東京', - 'expenses:office', - 'assets:cash:other' - ]); - expect(() => exports.take(2)).toThrow(); -}); - -it('drains ambiguous exports without projecting or storing them', () => { - const exports = new AccountTransactionExports('assets:cash', true); - const wire = json({ tindex: 7, tpostings: [{}] }); - exports.original(wire); - exports.transacted(wire); - expect([...exports.transactions()]).toEqual([]); - expect(() => exports.original('{')).toThrow(); -}); - -it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])( - 'rejects invalid index %s before storing', - (index) => { - const exports = new AccountTransactionExports('assets:cash', false); - expect(() => exports.original(json(native(index)))).toThrow(); - expect(() => exports.transacted(json(native(index)))).toThrow(); - } -); - -it('rejects duplicate originals, duplicate transacted records, and missing joins', () => { - const exports = new AccountTransactionExports('assets:cash', false); - exports.original(json(native())); - expect(() => exports.original(json(native()))).toThrow('Duplicate'); - expect(() => [...exports.transactions()]).toThrow('Missing native transacted'); - expect(() => exports.transacted(json(native(8)))).toThrow('Missing native original'); - exports.transacted(json(native())); - expect(() => exports.transacted(json(native()))).toThrow('Duplicate'); - expect(() => exports.take(8)).toThrow(); -}); - -it('does not attach mismatched header IDs or same-index changed metadata', () => { - const exports = new AccountTransactionExports('assets:cash', false); - exports.original(json(native())); - const changed = native(); - changed.tcomment += 'changed'; - expect(() => exports.transacted(json(changed))).toThrow('Mismatched'); -}); - -it('joins exports containing native scientific integer mantissas without floating-point loss', () => { - const exports = new AccountTransactionExports('assets:cash', false); - const wire = json(native()).replace( +it('decodes native numerical tokens exactly without projecting away unknown fields', () => { + const native = { ...transactionFixture(), futureNativeField: { retained: true } }; + const json = JSON.stringify(native).replace( '"decimalMantissa":-10000', - '"decimalMantissa":-9.00719925474099301e17' + '"decimalMantissa":-900719925474099301e2' ); - exports.original(wire); - exports.transacted(wire); - const posting = exports.take(7).postings[0]; - expect(posting.amounts[0].quantity).toBe('-9007199254740993.01'); - expect(posting.transactedAmounts).toEqual([ - { commodity: 'USD', quantity: '-9007199254740993.01' } - ]); + const decoded = decodeNativeTransaction(json); + expect(decoded.tpostings[0].pamount[0].aquantity.decimalMantissa).toBe(-90071992547409930100n); + expect(decoded).toMatchObject({ + tindex: native.tindex, + tdescription: native.tdescription, + ttags: native.ttags, + tsourcepos: native.tsourcepos, + futureNativeField: { retained: true } + }); }); -it.each(['original', 'transacted'] as const)( - 'rejects fractional scientific mantissas in %s exports', - (kind) => { - const exports = new AccountTransactionExports('assets:cash', false); - exports.original(json(native())); - const malformed = json(native()).replace('"decimalMantissa":-10000', '"decimalMantissa":1e-20'); - expect(() => exports[kind](malformed)).toThrow('integer mantissa'); - } -); +it('rejects output that is not a native transaction record', () => { + expect(() => decodeNativeTransaction('[]')).toThrow(); + expect(() => decodeNativeTransaction('{"tindex":1}')).toThrow(); + expect(() => decodeNativeTransaction('{"tindex":0,"tpostings":[]}')).toThrow(); +}); diff --git a/packages/hledger/src/transactions/export.ts b/packages/hledger/src/transactions/export.ts index c71c380..f52dd47 100644 --- a/packages/hledger/src/transactions/export.ts +++ b/packages/hledger/src/transactions/export.ts @@ -1,14 +1,5 @@ import { parseExactJson } from '../amounts.js'; -import { - prepareTransaction, - requireTransactionIndex, - type NativeTransaction, - type PreparedTransaction -} from './transaction.js'; - -type Entry = - | { kind: 'original'; transaction: NativeTransaction } - | { kind: 'joined'; transaction: PreparedTransaction }; +import { requireTransactionIndex, type NativeTransaction } from './transaction.js'; export function decodeNativeTransaction(json: string): NativeTransaction { const value = parseExactJson(json) as NativeTransaction; @@ -22,61 +13,3 @@ export function decodeNativeTransaction(json: string): NativeTransaction { requireTransactionIndex(value.tindex); return value; } - -/** One account only. Never retain the wider case-insensitive print selection. */ -export class AccountTransactionExports { - private readonly entries = new Map(); - private readonly account: string; - private readonly ambiguous: boolean; - - constructor(account: string, ambiguous: boolean) { - this.account = account; - this.ambiguous = ambiguous; - } - - private selected(transaction: NativeTransaction): boolean { - return transaction.tpostings.some(({ paccount }) => { - if (typeof paccount !== 'string') throw new Error('Invalid native posting account.'); - return paccount === this.account || paccount.startsWith(`${this.account}:`); - }); - } - - original(json: string): void { - const transaction = decodeNativeTransaction(json); - if (this.ambiguous || !this.selected(transaction)) return; - if (this.entries.has(transaction.tindex)) throw new Error('Duplicate native original index.'); - // Keep the native original until its non-amount structure can be checked. - this.entries.set(transaction.tindex, { kind: 'original', transaction }); - } - - transacted(json: string): void { - const native = decodeNativeTransaction(json); - if (this.ambiguous) return; - const entry = this.entries.get(native.tindex); - if (!entry) { - if (this.selected(native)) throw new Error('Missing native original index.'); - return; - } - if (entry.kind !== 'original') throw new Error('Duplicate native transacted index.'); - const transaction = prepareTransaction(entry.transaction, native); - // Replace the raw original only after validating the joined exports. - this.entries.set(native.tindex, { kind: 'joined', transaction }); - } - - *transactions(): IterableIterator { - for (const entry of this.entries.values()) { - if (entry.kind !== 'joined') throw new Error('Missing native transacted index.'); - yield entry.transaction; - } - } - - take(index: number): PreparedTransaction { - requireTransactionIndex(index); - const entry = this.entries.get(index); - if (!entry || entry.kind !== 'joined') { - throw new Error('Missing or duplicate native register index.'); - } - this.entries.delete(index); - return entry.transaction; - } -} diff --git a/packages/hledger/src/transactions/index.ts b/packages/hledger/src/transactions/index.ts index 2491f22..5c19f76 100644 --- a/packages/hledger/src/transactions/index.ts +++ b/packages/hledger/src/transactions/index.ts @@ -1,3 +1,4 @@ export { decodeNativeTransaction } from './export.js'; export { assertTransactionExportsMatch } from './transaction.js'; export type { NativeTransaction } from './transaction.js'; +export type { NativeTransactionPair, PreparationTransactions } from './store.js'; diff --git a/packages/hledger/src/transactions/investment.test.ts b/packages/hledger/src/transactions/investment.test.ts index 49aa7ad..a3787b9 100644 --- a/packages/hledger/src/transactions/investment.test.ts +++ b/packages/hledger/src/transactions/investment.test.ts @@ -112,7 +112,7 @@ function cashEvidence(commodity = 'USD'): PreparedTransaction { ]); } -function prepare( +async function prepare( input: Omit, related: PreparedTransaction[] = [], roles = metadata(), @@ -120,11 +120,11 @@ function prepare( ) { return prepareInvestmentActivity( input, - accountInvestmentContext([...related, input.transaction], roles, canonicalAccount) + await accountInvestmentContext([...related, input.transaction], roles, canonicalAccount) ); } -it('keeps externally funded two-security purchases together without duplicating account cash', () => { +it('keeps externally funded two-security purchases together without duplicating account cash', async () => { const input = row( [ posting('holdings', [security('VTI', '10.00')], 'acquire', [value('USD', '1000')]), @@ -137,7 +137,7 @@ it('keeps externally funded two-security purchases together without duplicating [amount('VTI', '10.00'), amount('BND', '5.00')], [amount('VTI', '10.00'), amount('BND', '5.00')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities).toEqual([ { commodity: 'VTI', @@ -167,7 +167,7 @@ it('keeps externally funded two-security purchases together without duplicating expect(result.issues).toEqual([]); }); -it('keeps internal cash change and the final native balance once, not per security', () => { +it('keeps internal cash change and the final native balance once, not per security', async () => { const input = row( [ posting('holdings', [security('VTI', '10.00')], 'acquire', [value('USD', '1000')]), @@ -178,7 +178,7 @@ it('keeps internal cash change and the final native balance once, not per securi [amount('VTI', '10.00'), amount('USD', '-1400.00'), amount('BND', '5.00')], [amount('VTI', '20.00'), amount('USD', '600.00'), amount('BND', '5.00')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities.map(({ action }) => action)).toEqual(['buy', 'buy']); expect(result.cash).toEqual({ direction: 'out', @@ -193,7 +193,7 @@ it('keeps internal cash change and the final native balance once, not per securi ); }); -it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separate', () => { +it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separate', async () => { const gain = posting('gains', [amount('USD', '-250')], 'gain'); gain.tags.push(['_generated-posting', '']); const input = row( @@ -208,7 +208,7 @@ it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separa [amount('VTI', '-5'), amount('USD', '750.00')], [amount('VTI', '15'), amount('USD', '950.00')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities[0]).toMatchObject({ action: 'sell', prices: [value('USD', '150.00')], @@ -223,7 +223,7 @@ it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separa it.each(['UnitCost', 'TotalCost'] as const)( 'retains native %s values without deriving a unit price', - (kind) => { + async (kind) => { const shares = security('VTI', '0.000000000000000001'); shares.cost = { kind, amount: amount('USD', '9007199254740993.0001') }; const input = row( @@ -234,7 +234,7 @@ it.each(['UnitCost', 'TotalCost'] as const)( [0], [amount('VTI', '0.000000000000000001')] ); - const result = prepare(input).securities[0]; + const result = (await prepare(input)).securities[0]; expect(result.transactedAmounts).toEqual([value('USD', '12345678901234567890.00000001')]); expect(result.prices).toEqual( kind === 'UnitCost' ? [value('USD', '9007199254740993.0001')] : [] @@ -262,7 +262,7 @@ it.each([ } ])( 'keeps native TotalCost $role value distinct from net settlement and expense', - ({ role, units, transacted, cash, expense, action }) => { + async ({ role, units, transacted, cash, expense, action }) => { // Values observed in the local native edge probe, not calculated by this preparation layer. const shares = security('XYZ', units, null, '40'); shares.cost = { kind: 'TotalCost', amount: amount('USD', transacted) }; @@ -277,7 +277,7 @@ it.each([ postings.push(gain); } const input = row(postings, [0, 1], [amount('XYZ', units), amount('USD', cash)]); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities).toEqual([ { commodity: 'XYZ', @@ -297,7 +297,7 @@ it.each([ } ); -it('keeps the native income/acquisition co-occurrence without manufacturing cash movement', () => { +it('keeps the native income/acquisition co-occurrence without manufacturing cash movement', async () => { const input = row( [ posting('holdings', [security('XYZ', '1', '50.00', '50.00')], 'acquire', [ @@ -309,7 +309,7 @@ it('keeps the native income/acquisition co-occurrence without manufacturing cash [amount('XYZ', '1')], [amount('USD', '895.00'), amount('XYZ', '4')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities[0]).toMatchObject({ action: 'acquire', prices: [value('USD', '50.00')], @@ -327,7 +327,7 @@ it('keeps the native income/acquisition co-occurrence without manufacturing cash expect(result.issues).toEqual(['unattributed-income']); }); -it('concatenates complete native value bags without zipping, summing or normalizing quantities', () => { +it('concatenates complete native value bags without zipping, summing or normalizing quantities', async () => { const input = row( [ posting('holdings', [security('BND', '1')], 'acquire', [value('USD', '10.0')]), @@ -344,7 +344,7 @@ it('concatenates complete native value bags without zipping, summing or normaliz [3, 2, 1, 0], [amount('VTI', '6.000'), amount('BND', '3'), amount('VTI', '0.00')] ); - const result = prepare(input); + const result = await prepare(input); expect( result.securities.map(({ commodity, postingIndexes }) => ({ commodity, postingIndexes })) ).toEqual([ @@ -363,23 +363,26 @@ it('concatenates complete native value bags without zipping, summing or normaliz it.each([ ['transfer-from', '-2.00', 'transfer-out'], ['transfer-to', '2.00', 'transfer-in'] -] as const)('keeps %s units distinct from transacted monetary values', (role, quantity, action) => { - const input = row( - [posting('holdings', [security('VTI', quantity, null)], role, [value('VTI', quantity)])], - [0], - [amount('VTI', quantity)] - ); - expect(prepare(input).securities[0]).toEqual({ - commodity: 'VTI', - action, - change: [value('VTI', quantity)], - postingIndexes: [0], - prices: [], - transactedAmounts: null - }); -}); +] as const)( + 'keeps %s units distinct from transacted monetary values', + async (role, quantity, action) => { + const input = row( + [posting('holdings', [security('VTI', quantity, null)], role, [value('VTI', quantity)])], + [0], + [amount('VTI', quantity)] + ); + expect((await prepare(input)).securities[0]).toEqual({ + commodity: 'VTI', + action, + change: [value('VTI', quantity)], + postingIndexes: [0], + prices: [], + transactedAmounts: null + }); + } +); -it('does not turn a native omitted converted posting into a zero or a computed value', () => { +it('does not turn a native omitted converted posting into a zero or a computed value', async () => { const input = row( [ posting('holdings', [security('VTI', '10')], 'acquire', null), @@ -388,7 +391,7 @@ it('does not turn a native omitted converted posting into a zero or a computed v [0], [amount('VTI', '10')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0]).toMatchObject({ action: 'acquire', transactedAmounts: null }); expect(result.issues).toContain('unavailable-transacted-values'); expect(input.transaction.postings[0].transactedAmounts).toBeNull(); @@ -396,7 +399,7 @@ it('does not turn a native omitted converted posting into a zero or a computed v it.each<{ missing: ReportedAmount[] | null }>([{ missing: null }, { missing: [] }])( 'does not attribute a partial value bag when any contributing posting is unavailable: %j', - ({ missing }) => { + async ({ missing }) => { const input = row( [ posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]), @@ -406,11 +409,11 @@ it.each<{ missing: ReportedAmount[] | null }>([{ missing: null }, { missing: [] [0, 1], [amount('VTI', '3')] ); - expect(prepare(input).securities[0].transactedAmounts).toBeNull(); + expect((await prepare(input)).securities[0].transactedAmounts).toBeNull(); } ); -it('does not apportion a converted mixed-security posting by original amount indexes', () => { +it('does not apportion a converted mixed-security posting by original amount indexes', async () => { const input = row( [ posting('holdings', [security('VTI', '10'), security('BND', '5', '80.00')], 'acquire', [ @@ -421,7 +424,7 @@ it('does not apportion a converted mixed-security posting by original amount ind [0], [amount('VTI', '10'), amount('BND', '5')] ); - const result = prepare(input); + const result = await prepare(input); expect( result.securities.map(({ action, transactedAmounts }) => ({ action, transactedAmounts })) ).toEqual([ @@ -431,7 +434,7 @@ it('does not apportion a converted mixed-security posting by original amount ind expect(result.issues).toContain('mixed-security-posting'); }); -it('also refuses attribution when a single posting mixes security and cash amounts', () => { +it('also refuses attribution when a single posting mixes security and cash amounts', async () => { const input = row( [ posting('holdings', [security('VTI', '1'), amount('USD', '-100')], 'acquire', [ @@ -441,12 +444,12 @@ it('also refuses attribution when a single posting mixes security and cash amoun [0], [amount('VTI', '1'), amount('USD', '-100')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities[0].transactedAmounts).toBeNull(); expect(result.issues).toContain('mixed-security-posting'); }); -it('retains all expense evidence without inventing commissions or security allocation', () => { +it('retains all expense evidence without inventing commissions or security allocation', async () => { const input = row( [ posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]), @@ -457,14 +460,14 @@ it('retains all expense evidence without inventing commissions or security alloc [0], [amount('VTI', '1')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.expensePostingIndexes).toEqual([2, 3]); expect(result.cashPostingIndexes).toEqual([1]); expect(result.securities[0].transactedAmounts).toEqual([value('USD', '100')]); expect(result.securities[0]).not.toHaveProperty('commission'); }); -it('leaves dividend-like cash receipts generic and unattributed', () => { +it('leaves dividend-like cash receipts generic and unattributed', async () => { const input = row( [posting('holdings', [amount('USD', '25.00')]), posting('revenue', [amount('USD', '-25.00')])], [0], @@ -472,7 +475,7 @@ it('leaves dividend-like cash receipts generic and unattributed', () => { [amount('USD', '225.00')] ); input.transaction.description = 'VTI Dividend'; - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.securities).toEqual([]); expect(result.incomePostingIndexes).toEqual([1]); expect(result.issues).toEqual(['unattributed-income']); @@ -484,7 +487,7 @@ it.each([ { role: 'dispose', units: '-1', transacted: '-100', counterpart: '100', cash: '5', expense: '-5' } ])( 'does not promote $role with equity/unknown funding merely because a cash fee leg exists', - ({ role, units, transacted, counterpart, cash, expense }) => { + async ({ role, units, transacted, counterpart, cash, expense }) => { const opening = posting('opening-or-gift', [amount('USD', counterpart)]); opening.tags.push(['type', 'E']); const input = row( @@ -497,7 +500,7 @@ it.each([ [0], [amount('VTI', units)] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0]).toMatchObject({ action: role, transactedAmounts: [value('USD', transacted)] @@ -507,11 +510,11 @@ it.each([ // An untyped counterpart is equally insufficient; descriptions and names cannot help. opening.tags = []; input.transaction.description = 'Buy VTI'; - expect(prepare(input).securities[0].action).toBe(role); + expect((await prepare(input)).securities[0].action).toBe(role); } ); -it('retains income plus acquisition without inventing reinvestment or cash funding allocation', () => { +it('retains income plus acquisition without inventing reinvestment or cash funding allocation', async () => { const input = row( [ posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]), @@ -521,14 +524,14 @@ it('retains income plus acquisition without inventing reinvestment or cash fundi [0], [amount('VTI', '1')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0].action).toBe('acquire'); expect(result.incomePostingIndexes).toEqual([1]); expect(result.cashPostingIndexes).toEqual([2]); expect(result.issues).toContain('unattributed-income'); }); -it('recognizes gain roles outside the native G set and excludes generated income', () => { +it('recognizes gain roles outside the native G set and excludes generated income', async () => { const generated = posting('revenue', [amount('USD', '-2')]); generated.tags.push(['_generated-posting', '']); const input = row( @@ -540,12 +543,12 @@ it('recognizes gain roles outside the native G set and excludes generated income [0], [amount('USD', '5')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.gainPostingIndexes).toEqual([1]); expect(result.incomePostingIndexes).toEqual([]); }); -it('requires account-local real Cash-type evidence, not currency spelling or price currency', () => { +it('requires account-local real Cash-type evidence, not currency spelling or price currency', async () => { const input = row( [ posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]), @@ -555,7 +558,7 @@ it('requires account-local real Cash-type evidence, not currency spelling or pri [amount('VTI', '1'), amount('USD', '-100')], [amount('USD', '200')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0].action).toBe('acquire'); expect(result.cash).toEqual({ direction: 'unknown', @@ -567,7 +570,7 @@ it('requires account-local real Cash-type evidence, not currency spelling or pri expect(result.cashPostingIndexes).toEqual([]); }); -it('does not infer non-lot securities from ticker, account names, prices or untrusted type tags', () => { +it('does not infer non-lot securities from ticker, account names, prices or untrusted type tags', async () => { const priced = amount('AAPL', '2'); priced.cost = { kind: 'UnitCost', amount: amount('USD', '100') }; const stock = posting('assets:investments:stocks', [priced], 'acquire', [value('USD', '200')]); @@ -577,13 +580,13 @@ it('does not infer non-lot securities from ticker, account names, prices or untr [0], [amount('AAPL', '2')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities).toEqual([]); expect(result.cash.unclassifiedChange).toEqual([value('AAPL', '2')]); expect(result.issues).toContain('unclassified-commodity'); }); -it('allows explicit processed basis plus a recognized role to establish an undeclared symbol', () => { +it('allows explicit processed basis plus a recognized role to establish an undeclared symbol', async () => { const input = row( [ posting('holdings', [security('not a ticker', '2')], 'acquire', [value('USD', '200')]), @@ -592,10 +595,13 @@ it('allows explicit processed basis plus a recognized role to establish an undec [0], [amount('not a ticker', '2')] ); - expect(prepare(input).securities[0]).toMatchObject({ commodity: 'not a ticker', action: 'buy' }); + expect((await prepare(input)).securities[0]).toMatchObject({ + commodity: 'not a ticker', + action: 'buy' + }); }); -it('treats evidenced FX as mixed cash, not a security just because it has a cost', () => { +it('treats evidenced FX as mixed cash, not a security just because it has a cost', async () => { const euros = amount('EUR', '90.00'); euros.cost = { kind: 'TotalCost', amount: amount('USD', '100.00') }; const input = row( @@ -604,7 +610,7 @@ it('treats evidenced FX as mixed cash, not a security just because it has a cost [amount('EUR', '90.00'), amount('USD', '-100.00')], [amount('EUR', '90.00')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities).toEqual([]); expect(result.cash.direction).toBe('mixed'); expect(result.cash.change).toEqual([value('EUR', '90.00'), value('USD', '-100.00')]); @@ -618,7 +624,7 @@ it.each([ ['holdings', 'dispose', '-2', [amount('VTI', '-1')]] ] as const)( 'does not label unsupported short evidence as an ordinary trade (%s/%s/%s)', - (account, role, quantity, balance) => { + async (account, role, quantity, balance) => { const input = row( [ posting(account, [security('VTI', quantity)], role, [value('USD', '-200')]), @@ -628,13 +634,13 @@ it.each([ [amount('VTI', quantity)], [...balance] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0].action).toBe('unknown'); expect(result.issues).toContain('unsupported-short'); } ); -it('does not treat liability cash as an asset even if native account-query sets overlap', () => { +it('does not treat liability cash as an asset even if native account-query sets overlap', async () => { const roles = metadata(); roles.assets = new Set([...roles.assets, 'shorts']); roles.cash = new Set([...roles.cash, 'shorts']); @@ -644,7 +650,7 @@ it('does not treat liability cash as an asset even if native account-query sets [amount('USD', '-10')], [amount('USD', '-30')] ); - const result = prepare(input, [], roles); + const result = await prepare(input, [], roles); expect(result.cash.change).toEqual([]); expect(result.cash.unclassifiedChange).toEqual([value('USD', '-10')]); expect(result.cashPostingIndexes).toEqual([1]); @@ -652,7 +658,7 @@ it('does not treat liability cash as an asset even if native account-query sets it.each(['VirtualPosting', 'BalancedVirtualPosting'])( 'does not promote %s security or mixed cash to ordinary activity', - (kind) => { + async (kind) => { const virtual = posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]); virtual.kind = kind; const input = row( @@ -665,7 +671,7 @@ it.each(['VirtualPosting', 'BalancedVirtualPosting'])( [amount('VTI', '1'), amount('USD', '-100')], [amount('USD', '20')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0].action).toBe('unknown'); expect(result.cash.change).toEqual([]); expect(result.cash.unclassifiedChange).toEqual([value('USD', '-100')]); @@ -682,7 +688,7 @@ it.each([ { account: 'elsewhere', kind: 'BalancedVirtualPosting', contaminated: false } ])( 'uses scoped virtual history for later cash bags: $account/$kind', - ({ account, kind, contaminated }) => { + async ({ account, kind, contaminated }) => { const virtual = posting(account, [amount('USD', '10.00')]); virtual.kind = kind; const prior = transaction([posting('holdings', [amount('USD', '0.00')]), virtual]); @@ -693,7 +699,7 @@ it.each([ [amount('USD', '5.00')], [amount('USD', '15.00')] ); - const context = accountInvestmentContext( + const context = await accountInvestmentContext( [cashEvidence(), prior, input.transaction], metadata(), 'holdings' @@ -727,7 +733,7 @@ it.each([ it.each(['liabilities', 'income', 'expenses', 'gains', 'unknown'] as const)( 'keeps later parent cash bags unclassified after selected %s history', - (role) => { + async (role) => { const roles = metadata(); roles.assets = new Set([...roles.assets, 'portfolio:cash']); roles.cash = new Set([...roles.cash, 'portfolio:cash']); @@ -747,7 +753,7 @@ it.each(['liabilities', 'income', 'expenses', 'gains', 'unknown'] as const)( [amount('USD', '100.00')] ); const before = structuredClone({ prior, input, roles }); - const result = prepare(input, [prior], roles, 'portfolio'); + const result = await prepare(input, [prior], roles, 'portfolio'); expect(result.cash).toEqual({ direction: 'unknown', change: [], @@ -769,7 +775,7 @@ it.each([ { account: 'elsewhere', contaminated: false } ])( 'scopes non-asset history to the canonical account/subtree: $account', - ({ account, contaminated }) => { + async ({ account, contaminated }) => { const roles = metadata(); roles.assets = new Set([...roles.assets, 'portfolio:cash']); roles.cash = new Set([...roles.cash, 'portfolio:cash']); @@ -781,7 +787,7 @@ it.each([ [amount('USD', '200.00')], [amount('USD', contaminated ? '100.00' : '200.00')] ); - const context = accountInvestmentContext([prior, input.transaction], roles, 'portfolio'); + const context = await accountInvestmentContext([prior, input.transaction], roles, 'portfolio'); expect(context.hasNonAssetAccountPostings).toBe(contaminated); const result = prepareInvestmentActivity(input, context); expect(result.cash.direction).toBe(contaminated ? 'unknown' : 'in'); @@ -791,11 +797,11 @@ it.each([ } ); -it('does not use virtual Cash postings as commodity evidence', () => { +it('does not use virtual Cash postings as commodity evidence', async () => { const virtual = posting('wallet', [amount('unusual cash unit', '10')]); virtual.kind = 'VirtualPosting'; const input = row([virtual], [0], [amount('unusual cash unit', '10')]); - const result = prepare(input); + const result = await prepare(input); expect(result.cashPostingIndexes).toEqual([]); expect(result.cash.unclassifiedChange).toEqual([value('unusual cash unit', '10')]); }); @@ -815,26 +821,26 @@ it.each<{ tags: [string, string][] }>([ ['_ptype', 'gain'] ] } -])('defers missing, unknown and conflicting security roles: %j', ({ tags }) => { +])('defers missing, unknown and conflicting security roles: %j', async ({ tags }) => { const stock = posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]); stock.tags = tags; const input = row([stock, posting('wallet', [amount('USD', '-100')])], [0], [amount('VTI', '1')]); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0].action).toBe('unknown'); expect(result.issues).toContain('unknown-native-role'); }); -it('does not silently let conflicting role tags establish a security from basis alone', () => { +it('does not silently let conflicting role tags establish a security from basis alone', async () => { const stock = posting('holdings', [security('undeclared', '1')], 'acquire'); stock.tags.push(['_ptype', 'dispose']); const input = row([stock], [0], [amount('undeclared', '1')]); - const result = prepare(input); + const result = await prepare(input); expect(result.securities).toEqual([]); expect(result.issues).toContain('unknown-native-role'); expect(result.cash.unclassifiedChange).toEqual([value('undeclared', '1')]); }); -it('marks opposing native actions mixed and uses only the whole-transaction net change', () => { +it('marks opposing native actions mixed and uses only the whole-transaction net change', async () => { const input = row( [ posting('holdings', [security('VTI', '-2', '110')], 'dispose', [value('USD', '-220')]), @@ -845,7 +851,7 @@ it('marks opposing native actions mixed and uses only the whole-transaction net [amount('VTI', '1.00')], [amount('VTI', '8.00')] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0]).toMatchObject({ action: 'mixed', change: [value('VTI', '1.00')], @@ -855,7 +861,7 @@ it('marks opposing native actions mixed and uses only the whole-transaction net expect(result.issues).toContain('mixed-security-actions'); }); -it('retains both sides of a subtree share transfer without fabricating a net zero', () => { +it('retains both sides of a subtree share transfer without fabricating a net zero', async () => { const input = row( [ posting('holdings', [security('VTI', '-2', null)], 'transfer-from'), @@ -864,7 +870,7 @@ it('retains both sides of a subtree share transfer without fabricating a net zer [0, 1], [] ); - const result = prepare(input); + const result = await prepare(input); expect(result.securities[0]).toMatchObject({ action: 'mixed', change: [], @@ -873,7 +879,7 @@ it('retains both sides of a subtree share transfer without fabricating a net zer expect(result.cash.change).toEqual([]); }); -it('keeps costless acquisitions and disposals neutral despite counterpart cash', () => { +it('keeps costless acquisitions and disposals neutral despite counterpart cash', async () => { const input = row( [ posting('holdings', [security('VTI', '-1', null)], 'dispose'), @@ -882,7 +888,7 @@ it('keeps costless acquisitions and disposals neutral despite counterpart cash', [0], [amount('VTI', '-1')] ); - expect(prepare(input).securities[0]).toMatchObject({ + expect((await prepare(input)).securities[0]).toMatchObject({ action: 'dispose', prices: [], transactedAmounts: null @@ -897,9 +903,9 @@ it.each([ [[amount('USD', '10.00'), amount('USD', '-1.00')], 'mixed'] ] as const)( 'reads cash direction from exact signs without aggregating: %j', - (change, direction) => { + async (change, direction) => { const input = row([posting('holdings', [amount('USD', '0')])], [0], [...change]); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.cash.direction).toBe(direction); expect(result.cash.change).toEqual( change.map(({ commodity, quantity }) => ({ commodity, quantity })) @@ -908,9 +914,9 @@ it.each([ } ); -it('keeps unidentified zero units and balances visible, without inventing a currency zero', () => { +it('keeps unidentified zero units and balances visible, without inventing a currency zero', async () => { const input = row([posting('holdings', [])], [0], [amount('', '0.00')], [amount('mystery', '2')]); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.cash).toEqual({ direction: 'unknown', change: [], @@ -920,19 +926,19 @@ it('keeps unidentified zero units and balances visible, without inventing a curr }); }); -it('does not let an unclassified balance alone change a known cash movement direction', () => { +it('does not let an unclassified balance alone change a known cash movement direction', async () => { const input = row( [posting('holdings', [amount('USD', '1')])], [0], [amount('USD', '1')], [amount('USD', '10'), amount('mystery', '2'), amount('VTI', '5')] ); - const result = prepare(input, [cashEvidence()]); + const result = await prepare(input, [cashEvidence()]); expect(result.cash.direction).toBe('in'); expect(result.cash.unclassifiedBalance).toEqual([value('mystery', '2')]); }); -it('builds context in one pass without mutating metadata and excludes later discovered securities from cash', () => { +it('builds context in one pass without mutating metadata and excludes later discovered securities from cash', async () => { const roles = metadata(); const before = structuredClone(roles); const early = transaction([ @@ -945,14 +951,14 @@ it('builds context in one pass without mutating metadata and excludes later disc yield early; yield later; } - const context = accountInvestmentContext(related(), roles, 'holdings'); + const context = await accountInvestmentContext(related(), roles, 'holdings'); expect(context.securityCommodities).toEqual(new Set(['VTI', 'BND', 'special'])); expect(context.cashCommodities).toEqual(new Set(['tokens'])); expect(roles).toEqual(before); expect(roles.lotCommodities.has('special')).toBe(false); }); -it('preparation leaves the source row, tags, annotations, amount bags and context untouched', () => { +it('preparation leaves the source row, tags, annotations, amount bags and context untouched', async () => { const input = row( [ posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100.0000')]), @@ -962,7 +968,7 @@ it('preparation leaves the source row, tags, annotations, amount bags and contex [amount('VTI', '1.0000')], [amount('VTI', '4.0000')] ); - const context = accountInvestmentContext([input.transaction], metadata(), 'holdings'); + const context = await accountInvestmentContext([input.transaction], metadata(), 'holdings'); const before = structuredClone({ input, context }); const result = prepareInvestmentActivity(input, context); expect({ input, context }).toEqual(before); diff --git a/packages/hledger/src/transactions/investment.ts b/packages/hledger/src/transactions/investment.ts index 3720881..cea87d8 100644 --- a/packages/hledger/src/transactions/investment.ts +++ b/packages/hledger/src/transactions/investment.ts @@ -108,16 +108,16 @@ function isAsset(posting: PreparedPosting, metadata: InvestmentMetadata): boolea } /** Only related original transactions establish cash units; annotations are not cash legs. */ -export function accountInvestmentContext( - transactions: Iterable, +export async function accountInvestmentContext( + transactions: Iterable | AsyncIterable, metadata: InvestmentMetadata, canonicalAccount: string -): AccountInvestmentContext { +): Promise { const securityCommodities = new Set(metadata.lotCommodities); const cashCandidates = new Set(); let hasVirtualAccountPostings = false; let hasNonAssetAccountPostings = false; - for (const transaction of transactions) { + for await (const transaction of transactions) { for (const posting of transaction.postings) { if ( posting.account === canonicalAccount || diff --git a/packages/hledger/src/transactions/store.ts b/packages/hledger/src/transactions/store.ts new file mode 100644 index 0000000..c1f15a0 --- /dev/null +++ b/packages/hledger/src/transactions/store.ts @@ -0,0 +1,13 @@ +export interface NativeTransactionPair { + originalJson: string; + transactedJson: string; +} + +/** Caller-owned storage for one explicit preparation; never a journal writer. */ +export interface PreparationTransactions { + insertOriginal(json: string): Promise; + attachTransacted(json: string): Promise; + assertComplete(): Promise; + read(nativeIndex: number): Promise; + accountHistory(accountName: string): AsyncIterable; +} diff --git a/packages/snapshots/src/reports/publication.test.ts b/packages/snapshots/src/reports/publication.test.ts index 3cec727..5cb2cd3 100644 --- a/packages/snapshots/src/reports/publication.test.ts +++ b/packages/snapshots/src/reports/publication.test.ts @@ -1,5 +1,6 @@ import * as databases from '@hledger-reports/db'; import * as reports from '@hledger-reports/db/reports'; +import { decodeNativeTransaction } from '@hledger-reports/hledger/transactions'; import * as crypto from 'node:crypto'; import * as fs from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -20,6 +21,10 @@ import { ready, preparedAccountRows as rows } from '../../../db/src/reports/data.fixture.js'; +import { + nativeTransactionFixture, + richNativeExportsFixture +} from '../../../db/src/transactions/native.fixture.js'; vi.mock('node:fs/promises', async (original) => ({ ...(await original()) @@ -164,6 +169,92 @@ it('commits and closes a complete private database before making its directory v expect((await fs.stat(join(publication.directory, 'reports.sqlite'))).mode & 0o777).toBe(0o600); }); +it('publishes the stage-bound native import without changing either raw export', async () => { + const pair = richNativeExportsFixture(); + const stage = await createReportStage(snapshots, older); + const { insertOriginal, attachTransacted, assertComplete, read, accountHistory } = + stage.transactions; + await insertOriginal(pair.originalJson); + await attachTransacted(pair.transactedJson); + await assertComplete(); + expect(await read(7)).toEqual(pair); + const history: Array = []; + for await (const transaction of accountHistory('assets:cash')) history.push(transaction); + expect(history).toEqual([pair]); + await stage.publish(fixture(), [], journal); + const publication = await selected(); + await expect(insertOriginal(pair.originalJson)).rejects.toSatisfy( + safeError('Unable to prepare reports') + ); + await stage.discard(); + const database = new DatabaseSync(join(publication.directory, 'reports.sqlite'), { + readOnly: true + }); + try { + expect( + database + .prepare( + 'SELECT native_index, original_json, transacted_json FROM transactions ORDER BY native_index' + ) + .all() + ).toEqual([ + { + native_index: 7, + original_json: pair.originalJson, + transacted_json: pair.transactedJson + } + ]); + } finally { + database.close(); + } +}); + +it('pages complete native history and lets caller cancellation run between batches', async () => { + const stage = await createReportStage(snapshots, older); + let abort: ReturnType | undefined; + try { + const expected = Array.from({ length: 52 }, (_, index) => index + 1); + for (const index of expected) { + const json = JSON.stringify(nativeTransactionFixture(index)); + await stage.transactions.insertOriginal(json); + await stage.transactions.attachTransacted(json); + } + await stage.transactions.assertComplete(); + const controller = new AbortController(); + const interrupted: number[] = []; + for await (const pair of stage.transactions.accountHistory('assets:cash')) { + if (controller.signal.aborted) break; + interrupted.push(decodeNativeTransaction(pair.originalJson).tindex); + if (interrupted.length === 1) abort = setImmediate(() => controller.abort()); + } + expect(interrupted.length).toBeGreaterThan(0); + expect(interrupted.length).toBeLessThan(expected.length); + const complete: number[] = []; + for await (const pair of stage.transactions.accountHistory('assets:cash')) + complete.push(decodeNativeTransaction(pair.originalJson).tindex); + expect(complete).toEqual(expected); + } finally { + if (abort) clearImmediate(abort); + await stage.discard(); + } +}); + +it('rejects an incomplete native import without replacing the last good publication', async () => { + await publish(); + const previous = await selected(); + const stage = await createReportStage(snapshots, newer); + try { + await stage.transactions.insertOriginal(richNativeExportsFixture().originalJson); + await expect(stage.publish(fixture(), [], journal)).rejects.toSatisfy( + safeError('Unable to prepare reports') + ); + expect(await selected()).toEqual(previous); + expect(await fs.readdir(join(snapshots, '.sets'))).toEqual([basename(previous.directory)]); + } finally { + await stage.discard(); + } +}); + it('distinguishes empty-ready accounts from ambiguous access without requiring empty callbacks', async () => { const stage = await createReportStage(snapshots, older); await stage.writeRows('empty callback', []); diff --git a/packages/snapshots/src/reports/publication.ts b/packages/snapshots/src/reports/publication.ts index 1b17456..cc0ac47 100644 --- a/packages/snapshots/src/reports/publication.ts +++ b/packages/snapshots/src/reports/publication.ts @@ -1,8 +1,10 @@ import type { OverviewData, PreparedAccountSummary, - PreparedAccountRow + PreparedAccountRow, + PreparationTransactions } from '@hledger-reports/hledger'; +import type { NativeTransactionPair } from '@hledger-reports/hledger/transactions'; import { createReportDatabase, openReadOnlyReportDatabase, @@ -29,9 +31,18 @@ import { type StoredAccount, type StoredReportData } from '@hledger-reports/db/reports'; +import { + insertNativeTransaction, + attachTransactedTransaction, + assertTransactionImportComplete, + readNativeTransaction, + readAccountTransactionBatch, + type StoredNativeTransaction +} from '@hledger-reports/db/transactions'; import { randomInt, randomUUID } from 'node:crypto'; import { lstat, mkdir, rename, rm, symlink, unlink } from 'node:fs/promises'; import { basename, dirname, join } from 'node:path'; +import { setImmediate } from 'node:timers/promises'; import { namesIn, privateDirectory } from '../files.js'; const runIdPattern = /^[A-Za-z0-9_-]{1,128}(?![\s\S])/; @@ -45,6 +56,7 @@ export interface ReportPublication extends StoredReportData { } export interface ReportStage { + transactions: PreparationTransactions; writeRows(name: string, rows: readonly PreparedAccountRow[]): Promise; publish( overview: OverviewData, @@ -54,6 +66,12 @@ export interface ReportStage { discard(): Promise; } +function nativePair(transaction: StoredNativeTransaction | null): NativeTransactionPair { + if (!transaction || transaction.transactedJson === null) + throw new Error('Native transaction exports are unavailable.'); + return { originalJson: transaction.originalJson, transactedJson: transaction.transactedJson }; +} + function requestCreationKey(token: string): string { // The separator in latestSet preserves request ordering for prefix run IDs. return setPattern.exec(token)![1]; @@ -234,6 +252,35 @@ export async function createReportStage( } } return { + transactions: { + insertOriginal(json) { + return operation(() => insertNativeTransaction(writer, json)); + }, + attachTransacted(json) { + return operation(() => attachTransactedTransaction(writer, json)); + }, + assertComplete() { + return operation(() => assertTransactionImportComplete(writer)); + }, + read(nativeIndex) { + return operation(async () => + nativePair(await readNativeTransaction(writer, nativeIndex)) + ); + }, + async *accountHistory(accountName) { + let afterNativeIndex = 0; + while (true) { + const batch = await operation(() => + readAccountTransactionBatch(writer, accountName, afterNativeIndex) + ); + if (batch.length === 0) return; + for (const transaction of batch) yield nativePair(transaction); + afterNativeIndex = batch[batch.length - 1].nativeIndex; + // SQLite runs synchronously; let caller abort callbacks run between batches. + await setImmediate(); + } + } + }, writeRows(name, rows) { return operation(async () => { if (typeof name !== 'string' || !Array.isArray(rows)) throw new Error(); @@ -292,6 +339,7 @@ export async function createReportStage( }); } if (remaining.size) throw new Error(); + await assertTransactionImportComplete(writer); await writeReportData(writer, { overview, accounts: catalog, journal }); await writeAccountActivitySummaries(writer, activitySummaries); await writer.commit().execute(); -- 2.54.0