diff --git a/.changeset/prepare-native-json-stream.md b/.changeset/prepare-native-json-stream.md new file mode 100644 index 0000000..32655c5 --- /dev/null +++ b/.changeset/prepare-native-json-stream.md @@ -0,0 +1,5 @@ +--- +'@hledger-reports/worker': patch +--- + +Prepare saved financial reports without injected output dividers, using shared transaction records for period counts. diff --git a/apps/worker/src/workflows/prepare-reports/run.test.ts b/apps/worker/src/workflows/prepare-reports/run.test.ts index addfb10..44355c4 100644 --- a/apps/worker/src/workflows/prepare-reports/run.test.ts +++ b/apps/worker/src/workflows/prepare-reports/run.test.ts @@ -55,6 +55,8 @@ const stage = { insertOriginal: vi.fn(), attachTransacted: vi.fn(), assertComplete: vi.fn(), + readSummary: vi.fn(), + countInPeriod: vi.fn(), read: vi.fn(), accountHistory: vi.fn() }, diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index f5a4df6..a9563bd 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -5,6 +5,7 @@ import * as activityIndexes from './migrations/002_account_activity_indexes.js'; import * as activitySummaries from './migrations/003_account_activity_summaries.js'; import * as nativeTransactions from './migrations/004_native_transactions.js'; import * as sharedAccountTransactions from './migrations/005_shared_account_transactions.js'; +import * as transactionDates from './migrations/006_transaction_dates.js'; import type { ReportDatabase } from './schema.js'; export async function runMigrations(database: Kysely): Promise { @@ -17,7 +18,8 @@ export async function runMigrations(database: Kysely): Promise): Promise { + await database.schema + .createIndex('transactions_date') + .on('transactions') + .column('transaction_date') + .execute(); +} diff --git a/packages/db/src/transactions/index.ts b/packages/db/src/transactions/index.ts index 16a7fd2..cd20f47 100644 --- a/packages/db/src/transactions/index.ts +++ b/packages/db/src/transactions/index.ts @@ -2,6 +2,8 @@ export { insertNativeTransaction, attachTransactedTransaction, readNativeTransaction, + readTransactionSummary, + countTransactionsInPeriod, readAccountTransactionBatch, assertTransactionImportComplete } from './queries.js'; diff --git a/packages/db/src/transactions/queries.test.ts b/packages/db/src/transactions/queries.test.ts index a25c67b..d1c3dec 100644 --- a/packages/db/src/transactions/queries.test.ts +++ b/packages/db/src/transactions/queries.test.ts @@ -12,9 +12,11 @@ import { import { assertTransactionImportComplete, attachTransactedTransaction, + countTransactionsInPeriod, insertNativeTransaction, readAccountTransactionBatch, readNativeTransaction, + readTransactionSummary, type StoredNativeTransaction } from './queries.js'; import { nativeTransactionFixture, richNativeExportsFixture } from './native.fixture.js'; @@ -51,13 +53,19 @@ afterEach(async () => { it('retains exact native exports and original posting order through completion and read-only reopen', async () => { const database = await createDatabase(); const exports = richNativeExportsFixture(); - const nextJson = JSON.stringify(nativeTransactionFixture(8)); + const nextJson = JSON.stringify({ + ...nativeTransactionFixture(8), + tdate: '2026-05-02', + tstatus: 'Unmarked' + }); const complete: StoredNativeTransaction = { nativeIndex: 7, date: '2026-05-01', status: 'Pending', ...exports }; + expect(await readTransactionSummary(database)).toEqual({ transactionCount: 0, lastDate: null }); + expect(await countTransactionsInPeriod(database, '2026-05-01', '2026-05-03')).toBe(0); await expect(assertTransactionImportComplete(database)).resolves.toBeUndefined(); const writer = await database.startTransaction().execute(); try { @@ -124,11 +132,19 @@ it('retains exact native exports and original posting order through completion a expect(await readNativeTransaction(reader, 7)).toEqual(complete); expect(await readNativeTransaction(reader, 8)).toEqual({ nativeIndex: 8, - date: '2026-05-01', - status: 'Pending', + date: '2026-05-02', + status: 'Unmarked', originalJson: nextJson, transactedJson: nextJson }); + expect(await readTransactionSummary(reader)).toEqual({ + transactionCount: 2, + lastDate: '2026-05-02' + }); + // Count each whole transaction once, by primary date rather than its posting dates. + expect(await countTransactionsInPeriod(reader, '2026-05-01', '2026-05-03')).toBe(2); + expect(await countTransactionsInPeriod(reader, '2026-05-01', '2026-05-02')).toBe(1); + expect(await countTransactionsInPeriod(reader, '2026-04-16', '2026-04-17')).toBe(0); expect(await readNativeTransaction(reader, 9)).toBeNull(); expect(await readAccountTransactionBatch(reader, 'equity:conversion', 0)).toEqual([complete]); await expect(assertTransactionImportComplete(reader)).resolves.toBeUndefined(); diff --git a/packages/db/src/transactions/queries.ts b/packages/db/src/transactions/queries.ts index d40ead8..b0729e1 100644 --- a/packages/db/src/transactions/queries.ts +++ b/packages/db/src/transactions/queries.ts @@ -1,6 +1,7 @@ import { assertTransactionExportsMatch, decodeNativeTransaction, + type NativeTransactionSummary, type NativeTransaction } from '@hledger-reports/hledger/transactions'; import type { Kysely } from 'kysely'; @@ -105,6 +106,50 @@ export async function attachTransactedTransaction( throw new Error('Transacted native transaction was not attached.'); } +function requireTransactionCount(count: number): number { + if (!Number.isSafeInteger(count) || count < 0) + throw new Error('Invalid native transaction count.'); + return count; +} + +function isPeriodDate(value: string): boolean { + return ( + typeof value === 'string' && + /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$/.test(value) && + new Date(`${value}T00:00:00Z`).toISOString().slice(0, 10) === value + ); +} + +export async function readTransactionSummary( + database: Kysely +): Promise { + const summary = await database + .selectFrom('transactions') + .select(({ fn }) => [ + fn.countAll().as('transactionCount'), + fn.max('transaction_date').as('lastDate') + ]) + .executeTakeFirstOrThrow(); + requireTransactionCount(summary.transactionCount); + return summary; +} + +export async function countTransactionsInPeriod( + database: Kysely, + begin: string, + endExclusive: string +): Promise { + if (!isPeriodDate(begin) || !isPeriodDate(endExclusive) || begin >= endExclusive) + throw new Error('Invalid native transaction period.'); + const result = await database + .selectFrom('transactions') + .select(({ fn }) => fn.countAll().as('transactionCount')) + .where('transaction_date', '>=', begin) + .where('transaction_date', '<', endExclusive) + .executeTakeFirstOrThrow(); + return requireTransactionCount(result.transactionCount); +} + export async function readNativeTransaction( database: Kysely, nativeIndex: number diff --git a/packages/hledger/src/accounts/balances.test.ts b/packages/hledger/src/accounts/balances.test.ts index 26e9641..c5d49bc 100644 --- a/packages/hledger/src/accounts/balances.test.ts +++ b/packages/hledger/src/accounts/balances.test.ts @@ -1,17 +1,24 @@ -import { describe, expect, it } from 'vitest'; -import { decodeAccountBalances, organizeAccountBalance, planAccountBalances } from './balances.js'; +import { describe, expect, it, vi } from 'vitest'; +import { ReportStreamReader } from '../report-stream.js'; +import { organizeAccountBalance, planAccountBalances, readAccountBalances } from './balances.js'; const asOf = '2026-04-02'; -const frame = (raw: string, valued = raw, prefix = 'overview') => - `${prefix}\n__HLEDGER_ACCOUNT_BALANCES__\n${raw}\n__HLEDGER_ACCOUNT_VALUATIONS__\n${valued}`; const empty = '[[],[]]'; const amount = (commodity: string, quantity: string) => ({ commodity, quantity }); +function reportReader(...documents: string[]): ReportStreamReader { + return new ReportStreamReader( + (async function* () { + for (const document of documents) yield Buffer.from(`${document}\n`); + })(), + () => {} + ); +} + describe('finite account balance reports', () => { - it('appends only two all-account native tree commands with the exact cutoff and literal target', () => { + it('plans two all-account native tree commands with the exact cutoff and literal target', () => { const commands = planAccountBalances({ asOf, baseCurrency: 'US D,quoted' }); const balance = [ - '--', 'balance', '--tree', '--no-elide', @@ -23,30 +30,26 @@ describe('finite account balance reports', () => { '-Ojson', 'not:date:2026-04-03..' ]; - expect(commands).toEqual([ - '--', - 'echo', - '__HLEDGER_ACCOUNT_BALANCES__', - ...balance, - '--', - 'echo', - '__HLEDGER_ACCOUNT_VALUATIONS__', - ...balance, - '--value=2026-04-02,US D,quoted' + expect(commands).toEqual([balance, [...balance, '--value=2026-04-02,US D,quoted']]); + expect(planAccountBalances({ asOf, baseCurrency: '' })).toEqual([ + balance, + [...balance, '--value=2026-04-02,'] ]); expect(planAccountBalances({ asOf: null, baseCurrency: '' })).toEqual([]); }); - it('decodes from the end losslessly, preserving cost-distinguished entries and exact full names', () => { + it('reads two native JSON documents losslessly, preserving bags and exact full names', async () => { const raw = `[[["Assets:Parent","Parent",1,[ {"acommodity":"ABC","aquantity":{"decimalMantissa":900719925474099301,"decimalPlaces":2},"acost":{"tag":"UnitCost","contents":{"acommodity":"USD","aquantity":{"decimalMantissa":10,"decimalPlaces":0}}}}, {"acommodity":"ABC","aquantity":{"decimalMantissa":3,"decimalPlaces":0},"acost":{"tag":"UnitCost","contents":{"acommodity":"USD","aquantity":{"decimalMantissa":12,"decimalPlaces":0}}}} ]],["assets:parent","parent",1,[]],["Assets:Unused","Unused",1,[]]],[]]`; - const prefix = - 'metadata path\n__HLEDGER_ACCOUNT_BALANCES__\nnot a report\n__HLEDGER_ACCOUNT_VALUATIONS__\noverview'; - const text = frame(raw, empty, prefix); - const decoded = decodeAccountBalances(text, asOf); - expect(decoded.overviewBatch).toBe(prefix); + const valued = `[[["Assets:Parent","Parent",1,[ + {"acommodity":"USD","aquantity":{"decimalMantissa":170,"decimalPlaces":0}} + ]]],[]]`; + const reader = reportReader(raw, valued); + const json = vi.spyOn(reader, 'json'); + const decoded = await readAccountBalances(reader, asOf); + expect(json).toHaveBeenCalledTimes(2); expect(decoded.balances.get('Assets:Parent')).toEqual([ amount('ABC', '9007199254740993.01'), amount('ABC', '3') @@ -54,23 +57,33 @@ describe('finite account balance reports', () => { expect(decoded.balances.get('assets:parent')).toEqual([]); expect(decoded.balances.get('Assets:Unused')).toEqual([]); expect(decoded.balances.has('Parent')).toBe(false); - // Retrying has no effect on input or earlier results. - expect(decodeAccountBalances(text, asOf)).toEqual(decoded); + expect(organizeAccountBalance('Assets:Parent', 'ready', asOf, 'USD', decoded)).toEqual({ + status: 'ready', + balances: [amount('ABC', '9007199254740993.01'), amount('ABC', '3')], + valuation: { amounts: [amount('USD', '170')], unconverted: [] } + }); + await reader.eof(); }); - it('requires complete framing and native rows rather than accepting partial or duplicate output', () => { - expect(() => decodeAccountBalances(frame(empty).slice(0, -1), asOf)).toThrow(); - expect(() => decodeAccountBalances('overview', asOf)).toThrow(); - expect(() => - decodeAccountBalances(frame('[[["a","a",0,[]],["a","a",0,[]]],[]]'), asOf) - ).toThrow(); - expect(() => decodeAccountBalances(frame('[[["a","a",0,null]],[]]'), asOf)).toThrow(); + it('rejects duplicate or blank full names and invalid native amount bags', async () => { + for (const raw of [ + '[[["a","a",0,[]],["a","a",0,[]]],[]]', + '[[["","a",0,[]]],[]]', + '[[["a","a",0,null]],[]]' + ]) { + await expect(readAccountBalances(reportReader(raw, empty), asOf)).rejects.toThrow(); + } + await expect( + readAccountBalances(reportReader(empty, '[[["a","a",0,[]]],null]'), asOf) + ).rejects.toThrow(); }); - it('retains zero and unnamed residual evidence only within the exact-name subtree', () => { - const balances = new Map([['Assets:Parent', []]]); + it('retains inclusive native bags and descendant residual evidence within the exact-name subtree', () => { + const balances = new Map([['Assets:Parent', [amount('ABC', '17')]]]); const valuations = new Map([ ['Assets:Parent', [amount('USD', '170')]], + ['Assets:Parent:One', [amount('ABC', '1')]], + ['Assets:Parent:Two', [amount('ABC', '-1')]], ['Assets:Parent:Zero', [amount('ZERO', '0'), amount('', '0')]], ['Assets:Parentage', [amount('SIBLING', '1')]], ['assets:parent:other', [amount('OTHERCASE', '1')]] @@ -79,18 +92,17 @@ describe('finite account balance reports', () => { organizeAccountBalance('Assets:Parent', 'ready', asOf, 'USD', { balances, valuations }) ).toEqual({ status: 'ready', - balances: [], - valuation: { amounts: [amount('USD', '170')], unconverted: ['ZERO', ''] } + balances: [amount('ABC', '17')], + valuation: { amounts: [amount('USD', '170')], unconverted: ['ABC', 'ZERO', ''] } }); }); - it('keeps qualification and dates authoritative, but rejects missing dated ready rows', () => { - const reports = decodeAccountBalances('unchanged', null); - expect(reports).toEqual({ - overviewBatch: 'unchanged', - balances: new Map(), - valuations: new Map() - }); + it('keeps qualification and dates authoritative without reading undated reports', async () => { + const reader = reportReader(); + const json = vi.spyOn(reader, 'json'); + const reports = await readAccountBalances(reader, null); + expect(json).not.toHaveBeenCalled(); + expect(reports).toEqual({ balances: new Map(), valuations: new Map() }); expect(organizeAccountBalance('a', 'ambiguous', null, 'USD', reports)).toEqual({ status: 'ambiguous' }); diff --git a/packages/hledger/src/accounts/balances.ts b/packages/hledger/src/accounts/balances.ts index f5252d5..3c0c262 100644 --- a/packages/hledger/src/accounts/balances.ts +++ b/packages/hledger/src/accounts/balances.ts @@ -1,35 +1,33 @@ import { parseExactJson, reportedAmounts, type Amount, type ReportedAmount } from '../amounts.js'; import type { NativeValue } from '../overview/report.js'; import { isoDate } from '../report-date.js'; +import type { ReportStreamReader } from '../report-stream.js'; export type PreparedBalance = | { status: 'ambiguous' } | { status: 'undated' } | { status: 'ready'; balances: ReportedAmount[]; valuation: NativeValue }; -interface AccountBalanceReports { +export interface AccountBalanceReports { balances: Map; valuations: Map; } -const balanceMarker = '\n__HLEDGER_ACCOUNT_BALANCES__\n'; -const valuationMarker = '\n__HLEDGER_ACCOUNT_VALUATIONS__\n'; const failureMessage = 'Invalid hledger account balance report.'; -/** Append two finite reports to the existing cached-journal financial plan. */ +/** Plan two finite native reports against the cached journal. */ export function planAccountBalances({ asOf, baseCurrency }: { asOf: string | null; baseCurrency: string; -}): string[] { +}): string[][] { if (asOf === null) return []; const end = new Date(`${asOf}T00:00:00Z`); end.setUTCDate(end.getUTCDate() + 1); const endExclusive = isoDate(end); const balance = [ - '--', 'balance', '--tree', '--no-elide', @@ -42,16 +40,12 @@ export function planAccountBalances({ `not:date:${endExclusive}..` ]; return [ - '--', - 'echo', - balanceMarker.trim(), - ...balance, - '--', - 'echo', - valuationMarker.trim(), - ...balance, - // The inferred commodity ID is literal, even when empty or containing spaces/commas. - `--value=${asOf},${baseCurrency}` + balance, + [ + ...balance, + // The inferred commodity ID is literal, even when empty or containing spaces/commas. + `--value=${asOf},${baseCurrency}` + ] ]; } @@ -82,21 +76,14 @@ function decodeBalanceReport(text: string): Map { return rows; } -/** Pure prefix decoding: safe to retry while the existing report stream grows. */ -export function decodeAccountBalances( - batch: string, +export async function readAccountBalances( + reader: ReportStreamReader, asOf: string | null -): AccountBalanceReports & { overviewBatch: string } { - if (asOf === null) return { overviewBatch: batch, balances: new Map(), valuations: new Map() }; - // Metadata can contain literal framing text; native JSON strings escape newlines. - const valuationSplit = batch.lastIndexOf(valuationMarker); - if (valuationSplit < 0) throw new Error(failureMessage); - const valuations = decodeBalanceReport(batch.slice(valuationSplit + valuationMarker.length)); - batch = batch.slice(0, valuationSplit); - const balanceSplit = batch.lastIndexOf(balanceMarker); - if (balanceSplit < 0) throw new Error(failureMessage); - const balances = decodeBalanceReport(batch.slice(balanceSplit + balanceMarker.length)); - return { overviewBatch: batch.slice(0, balanceSplit), balances, valuations }; +): Promise { + if (asOf === null) return { balances: new Map(), valuations: new Map() }; + const balances = decodeBalanceReport(await reader.json()); + const valuations = decodeBalanceReport(await reader.json()); + return { balances, valuations }; } /** Qualification belongs to the existing native account directory, not row presence. */ diff --git a/packages/hledger/src/journal-stats.test.ts b/packages/hledger/src/journal-stats.test.ts new file mode 100644 index 0000000..f02bd36 --- /dev/null +++ b/packages/hledger/src/journal-stats.test.ts @@ -0,0 +1,14 @@ +import { expect, it } from 'vitest'; +import { journalMetadata } from './journal-stats.js'; + +it('rejects an authoritative field embedded in a multiline native journal path', () => { + const stats = `Main file : /fictional/path +Base currency : EUR +/part.journal +Base currency : USD +Last txn : 2026-04-16 +Txns : 1 +Txns span : 2026-04-16 to 2026-04-17 +`; + expect(() => journalMetadata(stats)).toThrow('Invalid native journal stats.'); +}); diff --git a/packages/hledger/src/journal-stats.ts b/packages/hledger/src/journal-stats.ts index f3026df..aeebe62 100644 --- a/packages/hledger/src/journal-stats.ts +++ b/packages/hledger/src/journal-stats.ts @@ -16,9 +16,9 @@ function validDate(text: string): string { } // Read exact native fields, not delimiter markers that can occur in a Main file path. -function statsFields(stats: string, expected: number) { +function statsFields(stats: string) { const lines = stats.split(/\r?\n/); - const field = (name: string, preserveWhitespace = false): string[] => { + const field = (name: string, preserveWhitespace = false): string => { const pattern = new RegExp( preserveWhitespace ? `^${name}[ \t]*: (.*)$` : `^${name}[ \t]*:[ \t]*(.*)$` ); @@ -28,14 +28,14 @@ function statsFields(stats: string, expected: number) { if (/\p{Cc}/u.test(match[1])) throw new Error(failureMessage); return [preserveWhitespace ? match[1] : match[1].trim()]; }); - if (matches.length !== expected) throw new Error(failureMessage); - return matches; + if (matches.length !== 1) throw new Error(failureMessage); + return matches[0]; }; return { baseCurrency: field('Base currency', true), last: field('Last txn'), - counts: field('Txns').map(transactionCount), - spans: field('Txns span') + count: transactionCount(field('Txns')), + span: field('Txns span') }; } @@ -52,23 +52,24 @@ function transactionCount(text: string): number { return nonnegativeInteger(match[1]); } -function wholeJournalMetadata(fields: ReturnType): { +export function journalMetadata(stats: string): { asOf: string | null; journalCoverage: JournalCoverage; baseCurrency: string; } { + const fields = statsFields(stats); // Native prints one separator space after the colon. The rest is the literal ID, // including empty, 'none', commas, case and leading/trailing spaces. - const baseCurrency = fields.baseCurrency[0]; - const last = fields.last[0]; + const baseCurrency = fields.baseCurrency; + const last = fields.last; let asOf: string | null = null; if (last !== 'none') { const lastMatch = /^(\d{4}-\d{2}-\d{2})(?:[ \t]+\([^\r\n]*\))?$/.exec(last); if (!lastMatch) throw new Error(failureMessage); asOf = validDate(lastMatch[1]); } - const transactionCount = fields.counts[0]; - const span = fields.spans[0]; + const transactionCount = fields.count; + const span = fields.span; if (transactionCount === 0) { if (asOf !== null || !/^to(?:[ \t]+\([^\r\n]*\))?$/.test(span)) { throw new Error(failureMessage); @@ -95,17 +96,3 @@ function wholeJournalMetadata(fields: ReturnType): { journalCoverage: { transactionCount, firstDate, lastDate: isoDate(lastDate) } }; } - -export function journalMetadata(stats: string) { - return wholeJournalMetadata(statsFields(stats, 1)); -} - -// Whole stats followed by the finite period stats, before any JSON report frames. -// Filtered empty periods have explicit spans, unlike an empty whole journal. -export function journalMetadataWithCounts(stats: string, periodCount: number) { - const fields = statsFields(stats, periodCount + 1); - return { - metadata: wholeJournalMetadata(fields), - transactionCounts: fields.counts.slice(1) - }; -} diff --git a/packages/hledger/src/overview/batch.test.ts b/packages/hledger/src/overview/batch.test.ts index c072cc3..90157a7 100644 --- a/packages/hledger/src/overview/batch.test.ts +++ b/packages/hledger/src/overview/batch.test.ts @@ -1,8 +1,9 @@ import { expect, it } from 'vitest'; -import { decodeOverviewReports, planOverviewReports } from './batch.js'; +import { planOverviewReports, readOverviewReports } from './batch.js'; import { planIncomeRanges } from './ranges.js'; -import { journalMetadata, journalMetadataWithCounts } from '../journal-stats.js'; -import { balanceFrames, incomeFrames, periodStats } from '../../test-fixtures.js'; +import { journalMetadata } from '../journal-stats.js'; +import { ReportStreamReader } from '../report-stream.js'; +import { balanceReports, incomeReports } from '../../test-fixtures.js'; const stats = (asOf = '2026-04-16', currency = 'USD') => asOf === 'none' @@ -13,66 +14,83 @@ function nextDay(asOf: string) { date.setUTCDate(date.getUTCDate() + 1); return date.toISOString().slice(0, 10); } -const batch = (asOf = '2026-04-16') => - stats(asOf) + periodStats(asOf) + incomeFrames(asOf) + balanceFrames(asOf); -function decode(text = batch(), discovery = stats()) { - const metadata = journalMetadata(discovery); - return decodeOverviewReports(text, metadata, planOverviewReports(metadata)); +function readerFor(documents: string[]) { + return new ReportStreamReader( + (async function* () { + yield Buffer.from(documents.join('\n')); + })(), + () => {} + ); } -it('plans four statements and twelve exact-date sheets in one finite cached financial batch', () => { +it('plans four monthly statements followed by twelve historical exact-date valuations', () => { const plan = planOverviewReports(journalMetadata(stats())); - expect( - plan.incomeRanges.map(({ months, begin, endExclusive }) => ({ months, begin, endExclusive })) - ).toEqual([ - { months: 1, begin: '2026-04-01', endExclusive: '2026-04-17' }, - { months: 3, begin: '2026-02-01', endExclusive: '2026-04-17' }, - { months: 6, begin: '2025-11-01', endExclusive: '2026-04-17' }, - { months: 12, begin: '2025-05-01', endExclusive: '2026-04-17' } + expect(plan.incomeRanges.map(({ months }) => months)).toEqual([1, 3, 6, 12]); + expect(plan.commands.map(([command]) => command)).toEqual([ + ...Array(4).fill('incomestatement'), + ...Array(12).fill('balancesheet') ]); - expect(plan.commands.slice(0, 25)).toEqual([ - 'stats', - ...plan.incomeRanges.flatMap(({ begin, endExclusive }) => [ - '--', - 'stats', - '-b', - begin, - '-e', - endExclusive + const income = plan.commands[0]; + expect(income).toEqual( + expect.arrayContaining([ + '--change', + '--monthly', + '-Ojson', + 'not:type:G', + 'not:date:2026-04-17..' ]) + ); + expect(income.slice(income.indexOf('-b'), income.indexOf('-e') + 2)).toEqual([ + '-b', + '2026-04-01', + '-e', + '2026-05-01' ]); - const commands = plan.commands.join(' ').split(' -- echo ').slice(1); - expect(commands).toHaveLength(16); - for (const [index, range] of plan.incomeRanges.entries()) { - expect(commands[index]).toBe( - `__HLEDGER_INCOME_${range.months}__ -- incomestatement --flat --change --monthly --empty --depth=2 -b ${range.begin} -e 2026-05-01 -Ojson not:type:G not:date:2026-04-17..` - ); - } - expect(plan.snapshotPeriods.map(({ date }) => date)).toEqual([ - '2025-05-31', - '2025-06-30', - '2025-07-31', - '2025-08-31', - '2025-09-30', - '2025-10-31', - '2025-11-30', - '2025-12-31', - '2026-01-31', - '2026-02-28', - '2026-03-31', - '2026-04-16' + const latest = plan.commands[15]; + expect(latest).toEqual( + expect.arrayContaining([ + '--historical', + '--value=2026-04-16,USD', + '-Ojson', + 'not:date:2026-04-17..' + ]) + ); + expect(latest.slice(latest.indexOf('-b'), latest.indexOf('-e') + 2)).toEqual([ + '-b', + '2026-04-01', + '-e', + '2026-04-17' ]); - for (const [index, point] of plan.snapshotPeriods.entries()) { - expect(commands[index + 4]).toBe( - `__HLEDGER_NET_WORTH_${index + 1}__ -- balancesheet --historical --flat --empty --depth=1 -b ${point.begin} -e ${point.endExclusive} --value=${point.date},USD -Ojson not:date:${point.endExclusive}..` - ); - } - const decoded = decode(); - expect(decoded.incomeStatements.map(({ months }) => months)).toEqual([1, 3, 6, 12]); - expect(decoded.snapshots).toHaveLength(12); - expect(decoded.incomeStatements.map(({ transactionCount }) => transactionCount)).toEqual([ - 1, 3, 6, 12 + expect(plan.snapshotPeriods[10].date).toBe('2026-03-31'); +}); + +it('reads native JSON in plan order and attaches supplied counts and discovery coverage unchanged', async () => { + const metadata = journalMetadata(stats()); + const plan = planOverviewReports(metadata); + const statements = incomeReports(); + statements[0] = statements[0].replace( + '"prrTotal":[]', + '"prrTotal":[{"acommodity":"USD","aquantity":{"decimalMantissa":900719925474099306,"decimalPlaces":2}}]' + ); + const reader = readerFor([...statements, ...balanceReports()]); + const reports = await readOverviewReports(reader, metadata, plan, [1, 3, 6, 12]); + expect(reports.journalCoverage).toEqual(metadata.journalCoverage); + expect( + reports.incomeStatements.map(({ months, transactionCount }) => [months, transactionCount]) + ).toEqual([ + [1, 1], + [3, 3], + [6, 6], + [12, 12] ]); + expect(reports.incomeStatements[0].report.cbrTotals.prrTotal[0].aquantity.decimalMantissa).toBe( + 900719925474099306n + ); + expect(reports.snapshots.map(({ date }) => date)).toEqual( + plan.snapshotPeriods.map(({ date }) => date) + ); + expect(reports.snapshots[11].report.cbrDates[0][1].contents).toBe('2026-04-17'); + await reader.eof(); }); it.each([ @@ -87,7 +105,6 @@ it.each([ expect(plan.incomeRanges[3].endExclusive).toBe(cutoff); expect(plan.snapshotPeriods[10].date).toBe(previous); expect(plan.snapshotPeriods[11].date).toBe(asOf); - expect(decode(batch(asOf), stats(asOf)).asOf).toBe(asOf); } ); @@ -95,51 +112,29 @@ it('rejects a twelve-month window outside supported dates', () => { expect(() => planOverviewReports(journalMetadata(stats('0001-06-30')))).toThrow(); }); -it('omits all financial commands and dates for undated input', () => { +it('omits all financial commands and reads for undated input', async () => { const metadata = journalMetadata(stats('none')); const plan = planOverviewReports(metadata); - expect(plan).toEqual({ commands: ['stats'], incomeRanges: [], snapshotPeriods: [] }); - expect(decodeOverviewReports(stats('none'), metadata, plan)).toEqual({ + expect(plan).toEqual({ commands: [], incomeRanges: [], snapshotPeriods: [] }); + const reader = readerFor([]); + expect(await readOverviewReports(reader, metadata, plan, [])).toEqual({ ...metadata, incomeStatements: [], snapshots: [] }); - expect(() => decodeOverviewReports(stats(), metadata, plan)).toThrow(); + await reader.eof(); }); -it.each(['INCOME_3', 'NET_WORTH_6'])('requires financial framing for %s', (name) => { - expect(() => decode(batch().replace(`__HLEDGER_${name}__`, 'missing'))).toThrow(); -}); - -it('splits backwards despite literal markers in paths and escaped JSON strings', () => { - const markers = '\n__HLEDGER_INCOME_12__\n\n__HLEDGER_NET_WORTH_12__\n'; - const text = batch().replace('"cbrTotals":', `"ignored":${JSON.stringify(markers)},"cbrTotals":`); - const withPaths = text.replaceAll( - 'Base currency', - `Main file : /private/${markers}filename\nBase currency` - ); - expect(decode(withPaths).snapshots).toHaveLength(12); -}); - -it.each([ - batch().replace('__HLEDGER_INCOME_3__\n{', '__HLEDGER_INCOME_3__\nSECRET {'), - batch() + 'private tail', - batch().replace('Last txn : 2026-04-16', 'Last txn : 2026-04-15'), - batch().replace('Base currency : USD', 'Base currency : usd'), - batch().replace('Txns : 1 (0.0 per day)\n', ''), - batch().replace('Txns : 3 (0.0 per day)', 'Txns : 9007199254740992 (0.0 per day)'), - batch().replace('Base currency : USD', 'Base currency : US\u0000D'), - `Main file : /private/\nTxns : 999\nfilename\n${batch()}` -])( - 'rejects malformed financial framing, counts or changed discovery metadata (case %#)', - (text) => { - expect(() => decode(text)).toThrow(); - } -); - -it('accepts bounded empty filtered stats without accepting them as an empty whole journal', () => { - const filtered = - 'Base currency : USD\nLast txn : none\nTxns : 0\nTxns span : 2026-05-01 to 2026-06-01\n'; - expect(journalMetadataWithCounts(stats() + filtered, 1).transactionCounts).toEqual([0]); - expect(() => journalMetadata(filtered)).toThrow(); +it.each([1, 7])('rejects a malformed native financial document at index %i', async (index) => { + const metadata = journalMetadata(stats()); + const documents = [...incomeReports(), ...balanceReports()]; + documents[index] = '{"cbrDates":[],"cbrTotals":{},"cbrSubreports":null}'; + await expect( + readOverviewReports( + readerFor(documents), + metadata, + planOverviewReports(metadata), + [1, 3, 6, 12] + ) + ).rejects.toThrow(); }); diff --git a/packages/hledger/src/overview/batch.ts b/packages/hledger/src/overview/batch.ts index 2425590..12b8e28 100644 --- a/packages/hledger/src/overview/batch.ts +++ b/packages/hledger/src/overview/batch.ts @@ -1,6 +1,7 @@ import { parseBalanceSheet, type HledgerBalanceSheet } from './balance-sheet.js'; import { parseIncomeStatement, type HledgerIncomeStatement } from './income-statement.js'; -import { journalMetadataWithCounts, type JournalCoverage } from '../journal-stats.js'; +import type { journalMetadata, JournalCoverage } from '../journal-stats.js'; +import type { ReportStreamReader } from '../report-stream.js'; import { isoDate } from '../report-date.js'; import { planIncomeRanges, type IncomeRangePlan, type Period } from './ranges.js'; @@ -15,39 +16,17 @@ export interface OverviewReports { snapshots: (Period & { date: string; report: HledgerBalanceSheet })[]; } -// Static framing for four ranges and twelve valuations, not a report registry. -const incomeMarkers = [ - '\n__HLEDGER_INCOME_1__\n', - '\n__HLEDGER_INCOME_3__\n', - '\n__HLEDGER_INCOME_6__\n', - '\n__HLEDGER_INCOME_12__\n' -]; -const balanceMarkers = Array.from( - { length: 12 }, - (_, index) => `\n__HLEDGER_NET_WORTH_${index + 1}__\n` -); -const failureMessage = 'Invalid hledger Overview batch.'; - export function planOverviewReports({ asOf, baseCurrency }: Pick) { const incomeRanges = asOf === null ? [] : planIncomeRanges(asOf); const snapshotPeriods: (Period & { date: string })[] = []; - const commands = ['stats']; + const commands: string[][] = []; if (asOf === null) return { commands, incomeRanges, snapshotPeriods }; - // Keep all text stats before JSON so backwards JSON framing remains unambiguous, - // even when Main file contains literal newlines and report markers. - for (const range of incomeRanges) { - commands.push('--', 'stats', '-b', range.begin, '-e', range.endExclusive); - } const history = incomeRanges[3]; - for (const [index, range] of incomeRanges.entries()) { - commands.push( - '--', - 'echo', - incomeMarkers[index].trim(), - '--', + for (const range of incomeRanges) { + commands.push([ 'incomestatement', '--flat', '--change', @@ -62,7 +41,7 @@ export function planOverviewReports({ 'not:type:G', // Monthly expansion defeats positive date queries and a mid-month -e cutoff. `not:date:${range.endExclusive}..` - ); + ]); } for (const [index, period] of history.periods.entries()) { const monthEnd = new Date(`${period.endExclusive}T00:00:00Z`); @@ -73,11 +52,7 @@ export function planOverviewReports({ date: index === 11 ? asOf : isoDate(monthEnd) }; snapshotPeriods.push(point); - commands.push( - '--', - 'echo', - balanceMarkers[index].trim(), - '--', + commands.push([ 'balancesheet', '--historical', '--flat', @@ -91,47 +66,28 @@ export function planOverviewReports({ `--value=${point.date},${baseCurrency}`, '-Ojson', `not:date:${point.endExclusive}..` - ); + ]); } return { commands, incomeRanges, snapshotPeriods }; } -export function decodeOverviewReports( - batch: string, - { asOf, baseCurrency }: Pick, - { incomeRanges, snapshotPeriods }: ReturnType -): OverviewReports { +export async function readOverviewReports( + reader: ReportStreamReader, + metadata: ReturnType, + { incomeRanges, snapshotPeriods }: ReturnType, + transactionCounts: readonly number[] +): Promise { + const incomeStatements: OverviewReports['incomeStatements'] = []; + for (const [index, range] of incomeRanges.entries()) { + incomeStatements.push({ + ...range, + transactionCount: transactionCounts[index], + report: parseIncomeStatement(await reader.json()) + }); + } const snapshots: OverviewReports['snapshots'] = []; - const statements: (IncomeRangePlan & { report: HledgerIncomeStatement })[] = []; - if (asOf !== null) { - // Decode backwards: paths may contain earlier literal markers; JSON strings escape them. - for (let index = 11; index >= 0; index--) { - const marker = balanceMarkers[index]; - const split = batch.lastIndexOf(marker); - if (split < 0) throw new Error(failureMessage); - snapshots.unshift({ - ...snapshotPeriods[index], - report: parseBalanceSheet(batch.slice(split + marker.length)) - }); - batch = batch.slice(0, split); - } - for (let index = 3; index >= 0; index--) { - const marker = incomeMarkers[index]; - const split = batch.lastIndexOf(marker); - if (split < 0) throw new Error(failureMessage); - statements.unshift({ - ...incomeRanges[index], - report: parseIncomeStatement(batch.slice(split + marker.length)) - }); - batch = batch.slice(0, split); - } + for (const period of snapshotPeriods) { + snapshots.push({ ...period, report: parseBalanceSheet(await reader.json()) }); } - const { metadata, transactionCounts } = journalMetadataWithCounts(batch, asOf === null ? 0 : 4); - const incomeStatements = statements.map((statement, index) => ({ - ...statement, - transactionCount: transactionCounts[index] - })); - if (metadata.asOf !== asOf || metadata.baseCurrency !== baseCurrency) - throw new Error(failureMessage); return { ...metadata, incomeStatements, snapshots }; } diff --git a/packages/hledger/src/overview/net-worth.test.ts b/packages/hledger/src/overview/net-worth.test.ts index cd9fc37..478f3a8 100644 --- a/packages/hledger/src/overview/net-worth.test.ts +++ b/packages/hledger/src/overview/net-worth.test.ts @@ -4,12 +4,7 @@ import { join } from 'node:path'; import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { prepareReports, getHledgerVersion } from '../index.js'; import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; -import { - balanceFrames, - decodeOverviewFixture, - incomeFrames, - periodStats -} from '../../test-fixtures.js'; +import { balanceReports, decodeOverviewFixture, incomeReports } from '../../test-fixtures.js'; import { planOverviewReports } from './batch.js'; import { journalMetadata } from '../journal-stats.js'; @@ -43,28 +38,28 @@ const nativeAmount = (commodity: string, mantissa: number) => ({ }); const stats = (currency = 'USD') => `Base currency : ${currency}\nLast txn : 2026-04-16\nTxns : 12\nTxns span : 2025-05-01 to 2026-04-17\n`; -const frames = () => - balanceFrames() - .split(/\n__HLEDGER_NET_WORTH_\d+__\n/) - .slice(1) - .map( - (text) => - JSON.parse(text) as { - cbrDates: { tag: string; contents: string }[][]; - cbrTotals: { prrAmounts: ReturnType[][] }; - cbrSubreports: [ - string, - { prTotals: { prrAmounts: ReturnType[][] } }, - boolean - ][]; - } - ); -function batch(reports = frames(), currency = 'USD') { - return `${stats(currency)}${periodStats()}${incomeFrames()}${reports.map((report, index) => `\n__HLEDGER_NET_WORTH_${index + 1}__\n${JSON.stringify(report)}`).join('')}`; +const sheets = () => + balanceReports().map( + (text) => + JSON.parse(text) as { + cbrDates: { tag: string; contents: string }[][]; + cbrTotals: { prrAmounts: ReturnType[][] }; + cbrSubreports: [ + string, + { prTotals: { prrAmounts: ReturnType[][] } }, + boolean + ][]; + } + ); +function documents(reports = sheets()) { + return { + incomeStatements: incomeReports(), + snapshots: reports.map((report) => JSON.stringify(report)) + }; } it('uses the exact native net bag, retaining mixed amounts, order and explicit zero entries', () => { - const reports = frames(); + const reports = sheets(); const report = reports[11]; // Deliberately unrelated numbers: no JS accounting or net reconstruction. report.cbrTotals.prrAmounts = [ @@ -76,7 +71,7 @@ it('uses the exact native net bag, retaining mixed amounts, order and explicit z report.cbrSubreports[1][1].prTotals.prrAmounts = [ [nativeAmount('unpriced', 0), nativeAmount('only-debt', 100)] ]; - const data = decodeOverviewFixture(stats(), batch(reports)); + const data = decodeOverviewFixture(stats(), documents(reports)); expect(data.netWorth).toEqual({ currency: 'USD', points: dates.map((date, index) => ({ @@ -99,11 +94,11 @@ it('uses the exact native net bag, retaining mixed amounts, order and explicit z }); it.each(['', 'US,Dollar'])( - 'preserves exact native base currency %j between both passes and in arguments', + 'preserves exact native base currency %j in projections and arguments', (currency) => { - const data = decodeOverviewFixture(stats(currency), batch(frames(), currency)); + const data = decodeOverviewFixture(stats(currency), documents()); expect(data.netWorth?.currency).toBe(currency); - const args = planOverviewReports(journalMetadata(stats(currency))).commands; + const args = planOverviewReports(journalMetadata(stats(currency))).commands.flat(); expect(args.filter((arg) => arg.startsWith('--value='))).toEqual( dates.map((date) => `--value=${date},${currency}`) ); @@ -113,77 +108,62 @@ it.each(['', 'US,Dollar'])( ); it.each([ - ['missing dates', (r: ReturnType[number]) => r.cbrDates.pop()], - ['extra dates', (r: ReturnType[number]) => r.cbrDates.push(r.cbrDates[0])], + ['missing dates', (r: ReturnType[number]) => r.cbrDates.pop()], + ['extra dates', (r: ReturnType[number]) => r.cbrDates.push(r.cbrDates[0])], [ 'wrong start', - (r: ReturnType[number]) => (r.cbrDates[0][0].contents = '2026-01-02') + (r: ReturnType[number]) => (r.cbrDates[0][0].contents = '2026-01-02') ], [ 'wrong final end', - (r: ReturnType[number]) => (r.cbrDates[0][1].contents = '2026-05-01') + (r: ReturnType[number]) => (r.cbrDates[0][1].contents = '2026-05-01') ], [ 'non-exact date', - (r: ReturnType[number]) => (r.cbrDates[0][0].tag = 'Unbounded') + (r: ReturnType[number]) => (r.cbrDates[0][0].tag = 'Unbounded') ], - ['missing side', (r: ReturnType[number]) => r.cbrSubreports.pop()], + ['missing side', (r: ReturnType[number]) => r.cbrSubreports.pop()], [ 'wrong assets contribution', - (r: ReturnType[number]) => (r.cbrSubreports[0][2] = false) + (r: ReturnType[number]) => (r.cbrSubreports[0][2] = false) ], [ 'wrong liabilities contribution', - (r: ReturnType[number]) => (r.cbrSubreports[1][2] = true) + (r: ReturnType[number]) => (r.cbrSubreports[1][2] = true) ], - ['wrong side', (r: ReturnType[number]) => (r.cbrSubreports[1][0] = 'Equity')], - ['extra total column', (r: ReturnType[number]) => r.cbrTotals.prrAmounts.push([])], - ['missing total column', (r: ReturnType[number]) => r.cbrTotals.prrAmounts.pop()], + ['wrong side', (r: ReturnType[number]) => (r.cbrSubreports[1][0] = 'Equity')], + ['extra total column', (r: ReturnType[number]) => r.cbrTotals.prrAmounts.push([])], + ['missing total column', (r: ReturnType[number]) => r.cbrTotals.prrAmounts.pop()], [ 'missing assets column', - (r: ReturnType[number]) => r.cbrSubreports[0][1].prTotals.prrAmounts.pop() + (r: ReturnType[number]) => r.cbrSubreports[0][1].prTotals.prrAmounts.pop() ], [ 'extra liabilities column', - (r: ReturnType[number]) => r.cbrSubreports[1][1].prTotals.prrAmounts.push([]) + (r: ReturnType[number]) => r.cbrSubreports[1][1].prTotals.prrAmounts.push([]) ] ])('rejects %s in a middle balance sheet without publishing partial income', (_name, mutate) => { - const reports = frames(); + const reports = sheets(); mutate(reports[3]); - expect(() => decodeOverviewFixture(stats(), batch(reports))).toThrow(); + expect(() => decodeOverviewFixture(stats(), documents(reports))).toThrow(); }); it('rejects a malformed middle balance report', () => { - const text = batch().replace( - /(__HLEDGER_NET_WORTH_3__\n)[\s\S]*?(\n__HLEDGER_NET_WORTH_4__)/, - '$1{"cbrDates":[],"cbrSubreports":null}$2' - ); - expect(() => decodeOverviewFixture(stats(), text)).toThrow(); + const reports = documents(); + reports.snapshots[2] = '{"cbrDates":[],"cbrSubreports":null}'; + expect(() => decodeOverviewFixture(stats(), reports)).toThrow(); }); describe.each(['net', 'assets', 'liabilities'] as const)('malformed consumed %s bags', (side) => { it('rejects a null bag without publishing partial reports', () => { - const marker = '\n__HLEDGER_NET_WORTH_3__\n'; - const next = '\n__HLEDGER_NET_WORTH_4__\n'; - const text = batch(); - const start = text.indexOf(marker) + marker.length; - const end = text.indexOf(next); - const report = text.slice(start, end); - const key = side === 'net' ? 'cbrTotals' : 'prTotals'; - let seen = 0; - const malformed = report.replace( - new RegExp(`"${key}":\\{"prrAmounts":\\[\\[\\]\\]\\}`, 'g'), - (matched) => { - seen++; - return seen === (side === 'liabilities' ? 2 : 1) - ? `"${key}":{"prrAmounts":[null]}` - : matched; - } - ); - expect(malformed).not.toBe(report); - expect(() => - decodeOverviewFixture(stats(), text.slice(0, start) + malformed + text.slice(end)) - ).toThrow(); + const reports = sheets(); + const report = reports[2]; + let totals; + if (side === 'net') totals = report.cbrTotals; + else if (side === 'assets') totals = report.cbrSubreports[0][1].prTotals; + else totals = report.cbrSubreports[1][1].prTotals; + Reflect.set(totals, 'prrAmounts', [null]); + expect(() => decodeOverviewFixture(stats(), documents(reports))).toThrow(); }); }); diff --git a/packages/hledger/src/overview/ranges.test.ts b/packages/hledger/src/overview/ranges.test.ts index 7838539..bd557c4 100644 --- a/packages/hledger/src/overview/ranges.test.ts +++ b/packages/hledger/src/overview/ranges.test.ts @@ -3,25 +3,15 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest'; import { planIncomeRanges, organizeIncomeRange } from './ranges.js'; -import type { HledgerIncomeStatement } from './income-statement.js'; +import { parseIncomeStatement, type HledgerIncomeStatement } from './income-statement.js'; +import { incomeReports } from '../../test-fixtures.js'; import type { Amount } from '../amounts.js'; import { prepareReports, getHledgerVersion } from '../index.js'; import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; const plan = planIncomeRanges('2026-04-16')[3]; function statement(range = plan): HledgerIncomeStatement { - const totals = () => ({ prrAmounts: range.periods.map(() => []), prrTotal: [] }); - return { - cbrDates: range.periods.map(({ begin, endExclusive }) => [ - { tag: 'Exact', contents: begin }, - { tag: 'Exact', contents: endExclusive } - ]), - cbrTotals: { prrTotal: [] }, - cbrSubreports: [ - ['Revenues', { prTotals: totals(), prRows: [] }, true], - ['Expenses', { prTotals: totals(), prRows: [] }, false] - ] - }; + return parseIncomeStatement(incomeReports()[([1, 3, 6, 12] as const).indexOf(range.months)]); } const amount = (commodity: string, mantissa: bigint): Amount => ({ acommodity: commodity, diff --git a/packages/hledger/src/overview/report.test.ts b/packages/hledger/src/overview/report.test.ts index 75b5e87..c4218fc 100644 --- a/packages/hledger/src/overview/report.test.ts +++ b/packages/hledger/src/overview/report.test.ts @@ -4,23 +4,18 @@ import { join } from 'node:path'; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { prepareReports, getHledgerVersion } from '../index.js'; import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js'; -import { - balanceFrames, - decodeOverviewFixture, - incomeFrames, - periodStats -} from '../../test-fixtures.js'; +import { balanceReports, decodeOverviewFixture, incomeReports } from '../../test-fixtures.js'; const stats = 'Base currency : USD\nLast txn : 2026-04-16\nTxns : 12\nTxns span : 2025-05-01 to 2026-04-17\n'; const emptyStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to \n'; -const batch = (metadata = stats) => metadata + periodStats() + incomeFrames() + balanceFrames(); +const reports = () => ({ incomeStatements: incomeReports(), snapshots: balanceReports() }); afterEach(() => { vi.useRealTimers(); }); it('publishes all four ranges, a shared twelve-month history and latest native sides', () => { - const data = decodeOverviewFixture(stats, batch()); + const data = decodeOverviewFixture(stats, reports()); expect(data).toMatchObject({ asOf: '2026-04-16', journalCoverage: { transactionCount: 12, firstDate: '2025-05-01', lastDate: '2026-04-16' }, @@ -49,19 +44,23 @@ it('publishes monthly history only from the twelve-month statement', () => { acommodity: 'USD', aquantity: { decimalMantissa: mantissa, decimalPlaces: 2 } }); - const [shorter, history] = incomeFrames().split('\n__HLEDGER_INCOME_12__\n'); - const statements = - shorter.replace('"prrAmounts":[[]]', `"prrAmounts":[[${amount(111)}]]`) + - '\n__HLEDGER_INCOME_12__\n' + - history.replace('"prrAmounts":[[]', `"prrAmounts":[[${amount(222)}]`); - const data = decodeOverviewFixture(stats, stats + periodStats() + statements + balanceFrames()); + const native = reports(); + native.incomeStatements[0] = native.incomeStatements[0].replace( + '"prrAmounts":[[]]', + `"prrAmounts":[[${amount(111)}]]` + ); + native.incomeStatements[3] = native.incomeStatements[3].replace( + '"prrAmounts":[[]', + `"prrAmounts":[[${amount(222)}]` + ); + const data = decodeOverviewFixture(stats, native); expect(data.incomeExpenses.months[0].income).toEqual([{ commodity: 'USD', quantity: '2.22' }]); expect(data.incomeExpenses.months[11].income).toEqual([]); expect(data.ranges![12].income).toEqual([]); }); it('projects undated input without invented ranges, sides, currency, dates or a timestamp', () => { - expect(decodeOverviewFixture(emptyStats, emptyStats)).toEqual({ + expect(decodeOverviewFixture(emptyStats, { incomeStatements: [], snapshots: [] }, [])).toEqual({ asOf: null, journalCoverage: { transactionCount: 0, firstDate: null, lastDate: null }, ranges: null, @@ -71,15 +70,19 @@ it('projects undated input without invented ranges, sides, currency, dates or a }); }); -it('uses second cached metadata and publishes no private paths or unused native fields', () => { - const metadata = `Main file : /private/SECRET.journal\n${stats - .replace('Txns : 12', 'Txns : 13') - .replace('2025-05-01 to', '2025-01-01 to')}`; - const text = batch(metadata).replace('"cbrTotals":', '"sourcePath":"SECRET","cbrTotals":'); - const data = decodeOverviewFixture(stats, text); +it('publishes no private metadata paths or unused native fields', () => { + const metadata = `Main file : /private/SECRET.journal\n${stats}`; + const native = reports(); + native.incomeStatements = native.incomeStatements.map((json) => + json.replace('"cbrTotals":', '"sourcePath":"SECRET","cbrTotals":') + ); + native.snapshots = native.snapshots.map((json) => + json.replace('"cbrTotals":', '"sourcePath":"SECRET","cbrTotals":') + ); + const data = decodeOverviewFixture(metadata, native); expect(data.journalCoverage).toEqual({ - transactionCount: 13, - firstDate: '2025-01-01', + transactionCount: 12, + firstDate: '2025-05-01', lastDate: '2026-04-16' }); expect(JSON.stringify(data)).not.toContain('SECRET'); @@ -88,7 +91,7 @@ it('uses second cached metadata and publishes no private paths or unused native it('does not consult the clock or assign the producer-owned completion timestamp', () => { vi.useFakeTimers({ toFake: ['Date'] }); vi.setSystemTime(new Date('2099-12-31T12:34:56.789Z')); - const data = decodeOverviewFixture(stats, batch()); + const data = decodeOverviewFixture(stats, reports()); expect(data.asOf).toBe('2026-04-16'); expect(data).not.toHaveProperty('preparedAt'); expect(data).not.toHaveProperty('hledgerVersion'); diff --git a/packages/hledger/src/prepare-reports/batch.test.ts b/packages/hledger/src/prepare-reports/batch.test.ts index ddf8db3..0564b83 100644 --- a/packages/hledger/src/prepare-reports/batch.test.ts +++ b/packages/hledger/src/prepare-reports/batch.test.ts @@ -4,7 +4,9 @@ import { planPreparedReportCommands } from './batch.js'; it('exports the whole journal once regardless of account count or ambiguous selections', () => { const { args } = planPreparedReportCommands([], { - journal: journalMetadata('Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'), + journal: journalMetadata( + 'Base currency : USD\nLast txn : 2026-04-16\nTxns : 1\nTxns span : 2026-04-16 to 2026-04-17\n' + ), accounts: [ { name: 'assets:cash', access: 'ready' }, { name: 'assets:cash:daily', access: 'ready' }, @@ -21,7 +23,7 @@ it('exports the whole journal once regardless of account count or ambiguous sele } }); expect(args.filter((argument) => argument === 'print')).toHaveLength(2); - expect(args.slice(args.indexOf('print'), args.indexOf('aregister'))).toEqual([ + expect(args.slice(args.indexOf('print'), args.indexOf('incomestatement'))).toEqual([ 'print', '--round=none', '-Ojson', @@ -34,4 +36,6 @@ it('exports the whole journal once regardless of account count or ambiguous sele ]); expect(args.filter((argument) => argument === 'aregister')).toHaveLength(2); expect(args).not.toContain('^assets:Case$'); + expect(args).not.toContain('echo'); + expect(args).not.toContain('stats'); }); diff --git a/packages/hledger/src/prepare-reports/batch.ts b/packages/hledger/src/prepare-reports/batch.ts index 4f00e23..3f52743 100644 --- a/packages/hledger/src/prepare-reports/batch.ts +++ b/packages/hledger/src/prepare-reports/batch.ts @@ -1,11 +1,12 @@ -import { decodeOverviewReports, planOverviewReports } from '../overview/batch.js'; +import { readOverviewReports, planOverviewReports } from '../overview/batch.js'; import { organizeOverviewData } from '../overview/report.js'; import { planAccountBalances, - decodeAccountBalances, + readAccountBalances, organizeAccountBalance, type PreparedBalance } from '../accounts/balances.js'; +import type { ReportStreamReader } from '../report-stream.js'; import type { PreparationMetadata, QualifiedAccount } from './metadata.js'; export interface PreparedAccountSummary extends QualifiedAccount { @@ -13,13 +14,6 @@ export interface PreparedAccountSummary extends QualifiedAccount { summary: PreparedBalance; } -// Keep the legacy delimiter for the finite financial prefix. -export const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n'; - -function echo(marker: string): string[] { - return ['--', 'echo', marker.trimEnd()]; -} - function escapeAccountName(name: string): string { return name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } @@ -29,41 +23,31 @@ export function planPreparedReportCommands( metadata: PreparationMetadata ) { const overviewPlan = planOverviewReports(metadata.journal); - const args = [ - ...inputFlags, - 'run', - '--', + const commands = [ + ['print', '--round=none', '-Ojson'], + ['print', '--round=none', '--value=transacted', '-Ojson'], ...overviewPlan.commands, - ...planAccountBalances(metadata.journal), - ...echo(financialEndMarker), - '--', - 'print', - '--round=none', - '-Ojson', - '--', - 'print', - '--round=none', - '--value=transacted', - '-Ojson' + ...planAccountBalances(metadata.journal) ]; for (const { name, access } of metadata.accounts) { if (access === 'ambiguous') continue; const exactAccountPattern = `^${escapeAccountName(name)}$`; - // No separator before the literal pattern: run owns `--` separators. - args.push('--', 'aregister', '--historical', '--empty', '-Ojson', exactAccountPattern); + commands.push(['aregister', '--historical', '--empty', '-Ojson', exactAccountPattern]); } + const args = [...inputFlags, 'run', ...commands.flatMap((command) => ['--', ...command])]; return { args, overviewPlan }; } -export function decodePreparedPrefix( - prefix: string, +export async function readPreparedReports( + reader: ReportStreamReader, metadata: PreparationMetadata, - overviewPlan: ReturnType + overviewPlan: ReturnType, + transactionCounts: readonly number[] ) { - const accountReports = decodeAccountBalances(prefix, metadata.journal.asOf); const overview = organizeOverviewData( - decodeOverviewReports(accountReports.overviewBatch, metadata.journal, overviewPlan) + await readOverviewReports(reader, metadata.journal, overviewPlan, transactionCounts) ); + const accountReports = await readAccountBalances(reader, metadata.journal.asOf); const accounts: PreparedAccountSummary[] = metadata.accounts.map(({ name, access }) => { const rowCount = access === 'ambiguous' ? null : 0; const summary = organizeAccountBalance( diff --git a/packages/hledger/src/prepare-reports/run.test.ts b/packages/hledger/src/prepare-reports/run.test.ts index 2ab5ad4..2311f58 100644 --- a/packages/hledger/src/prepare-reports/run.test.ts +++ b/packages/hledger/src/prepare-reports/run.test.ts @@ -1,5 +1,6 @@ import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { balanceReports, incomeReports } from '../../test-fixtures.js'; import { supportedHledgerVersion } from '../checks.js'; import { transactionFixture } from '../transactions/transaction.fixture.js'; import { prepareReports } from './run.js'; @@ -35,15 +36,33 @@ afterEach(async () => { }); const versionOutput = `hledger ${supportedHledgerVersion}, SECRET banner\n`; -const journalStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'; -const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n'; -const financialPrefix = `${journalStats}${financialEndMarker}`; +const emptyStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'; +const asOf = '2031-01-01'; + +function metadataStats(count: number): string { + return count === 0 + ? emptyStats + : `Base currency : USD\nLast txn : ${asOf}\nTxns : ${count}\nTxns span : 2031-01-01 to 2031-01-02\n`; +} + +interface NativeFixture { + output: string; + count: number; +} const nativeTransaction = { ...transactionFixture(), tdate: '2031-01-01', tdescription: 'λ, test' }; -const globalExports = `${financialPrefix}[${JSON.stringify(nativeTransaction)}]\n[${JSON.stringify(nativeTransaction)}]\n`; +const globalExports = `[${JSON.stringify(nativeTransaction)}]\n[${JSON.stringify(nativeTransaction)}]\n`; +const accountBalance = JSON.stringify([[['assets:cash', 'assets:cash', 0, []]], []]); +const financialReports = [ + ...incomeReports(asOf), + ...balanceReports(asOf), + accountBalance, + accountBalance +].join('\n'); +const datedExports = `${globalExports}${financialReports}\n`; const registerRow = JSON.stringify([ nativeTransaction, nativeTransaction, @@ -53,7 +72,7 @@ const registerRow = JSON.stringify([ [] ]); -function accountProtocol(rowCount: number): string { +function accountFixture(rowCount: number): NativeFixture { const transactions = Array.from({ length: rowCount }, (_, index) => ({ ...nativeTransaction, tindex: index + 1, @@ -66,7 +85,10 @@ function accountProtocol(rowCount: number): string { const rows = JSON.stringify( transactions.map((transaction) => [transaction, transaction, false, ['other, 東京'], [], []]) ); - return `${financialPrefix}${prints}\n${prints}\n${rows}`; + let output = `${prints}\n${prints}\n`; + if (rowCount > 0) output += `${financialReports}\n`; + output += rows; + return { output, count: rowCount }; } function startFixtureChild(script: string): ChildProcessWithoutNullStreams { @@ -86,8 +108,8 @@ function queueChildOutput(output: string) { }); } -function queuePreparationMetadata() { - queueChildOutput(journalStats); +function queuePreparationMetadata(count: number) { + queueChildOutput(metadataStats(count)); queueChildOutput('assets:cash\n'); queueChildOutput('assets:cash ; type: C\n'); queueChildOutput(''); @@ -95,23 +117,23 @@ function queuePreparationMetadata() { } function mockPreparationProcesses( - output: string, + fixture: NativeFixture, { exitCode = 0, keepOpen = false }: { exitCode?: number; keepOpen?: boolean } = {} ): ChildProcessWithoutNullStreams { queueChildOutput(versionOutput); - queuePreparationMetadata(); + queuePreparationMetadata(fixture.count); const child = startFixtureChild( `process.stdin.pipe(process.stdout,{end:false});process.stdin.on('end',()=>process.stdout.write('',()=>{${keepOpen ? 'setInterval(()=>{},1000)' : `process.exit(${exitCode})`}}));` ); processMock.spawn.mockImplementationOnce(() => { - child.stdin.end(output); + child.stdin.end(fixture.output); return child; }); return child; } -async function expectRejected(output: string, exitCode = 0) { - const child = mockPreparationProcesses(output, { exitCode }); +async function expectRejected(fixture: NativeFixture, exitCode = 0) { + const child = mockPreparationProcesses(fixture, { exitCode }); const writeRows = vi.fn(async () => {}); const error = await prepareReports('/private/SECRET', { transactions: new MemoryPreparationTransactions(), @@ -125,21 +147,28 @@ async function expectRejected(output: string, exitCode = 0) { } describe('preparation completion and failure', () => { - it('timestamps completion after the final row callback and stores only the checked version', async () => { + it('timestamps completion after native close and the final row callback, storing only the checked version', async () => { vi.useFakeTimers({ toFake: ['Date'] }); try { vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); - mockPreparationProcesses(`${globalExports}[${registerRow}] \t\r\n`); + const child = mockPreparationProcesses({ + output: `${datedExports}[${registerRow}] \t\r\n`, + count: 1 + }); + child.once('close', () => { + vi.setSystemTime(new Date('2026-01-01T00:01:00Z')); + }); const result = await prepareReports('/private', { transactions: new MemoryPreparationTransactions(), async writeRows() { + await children.get(child); vi.setSystemTime(new Date('2026-01-01T00:02:30Z')); } }); expect(result.overview.preparedAt).toBe('2026-01-01T00:02:30.000Z'); expect(result.overview.hledgerVersion).toBe(supportedHledgerVersion); expect(result.accounts).toMatchObject([ - { name: 'assets:cash', access: 'ready', rowCount: 1, summary: { status: 'undated' } } + { name: 'assets:cash', access: 'ready', rowCount: 1, summary: { status: 'ready' } } ]); expect(processMock.spawn).toHaveBeenCalledTimes(7); const inputFlags = ['--no-conf', '--lang=en', '--strict', '-f', '/private']; @@ -152,24 +181,70 @@ describe('preparation completion and failure', () => { } }); - it.each([ - { name: 'truncated account output', output: `${globalExports}[${registerRow}` }, - { - name: 'malformed native JSON between global exports', - output: `${financialPrefix}[${JSON.stringify(nativeTransaction)}]invalid[]` - } - ])('rejects $name without returning a partial report', async ({ output }) => { - await expectRejected(output); + it('prepares an undated journal without consuming overview or balance documents', async () => { + mockPreparationProcesses(accountFixture(0)); + const writeRows = vi.fn(async () => {}); + const result = await prepareReports('/private', { + transactions: new MemoryPreparationTransactions(), + writeRows + }); + expect(result.accounts).toMatchObject([ + { name: 'assets:cash', access: 'ready', rowCount: 0, summary: { status: 'undated' } } + ]); + expect(writeRows).toHaveBeenCalledExactlyOnceWith('assets:cash', []); + expect(processMock.spawn).toHaveBeenCalledTimes(7); }); + it.each([ + { name: 'truncated account output', output: `${datedExports}[${registerRow}` }, + { name: 'an object instead of a global transaction array', output: '{}\n[]\n' }, + { + name: 'malformed native JSON between global exports', + output: `[${JSON.stringify(nativeTransaction)}]invalid[]` + } + ])('rejects $name without returning a partial report', async ({ output }) => { + await expectRejected({ output, count: 1 }); + }); + + it('rejects an unexpected native JSON document after the complete report plan', async () => { + const fixture = accountFixture(1); + await expectRejected({ ...fixture, output: `${fixture.output}\n{}` }); + }); + + it.each([ + { name: 'transaction count', count: 2, transaction: nativeTransaction }, + { + name: 'maximum primary date', + count: 1, + transaction: { ...nativeTransaction, tdate: '2030-12-31' } + } + ])( + 'rejects a metadata/global $name mismatch before financial reports or rows', + async ({ count, transaction }) => { + const print = JSON.stringify([transaction]); + mockPreparationProcesses({ output: `${print}\n${print}\n`, count }); + const transactions = new MemoryPreparationTransactions(); + const readSummary = vi.spyOn(transactions, 'readSummary'); + const countInPeriod = vi.spyOn(transactions, 'countInPeriod'); + const writeRows = vi.fn(async () => {}); + await expect(prepareReports('/private', { transactions, writeRows })).rejects.toThrow(); + expect(readSummary).toHaveBeenCalledOnce(); + expect(countInPeriod).not.toHaveBeenCalled(); + expect(writeRows).not.toHaveBeenCalled(); + } + ); + it('rejects a repeated register transaction even though shared storage permits rereads', async () => { - const writes = await expectRejected(`${globalExports}[${registerRow},${registerRow}]`); + const writes = await expectRejected({ + output: `${datedExports}[${registerRow},${registerRow}]`, + count: 1 + }); expect(writes).not.toHaveBeenCalled(); }); it('rejects incomplete global pairs even when every account is ambiguous', async () => { queueChildOutput(versionOutput); - queueChildOutput(journalStats); + queueChildOutput(metadataStats(1)); queueChildOutput('assets:case\nassets:Case\n'); queueChildOutput('assets:case ; type: C\nassets:Case ; type: C\n'); queueChildOutput(''); @@ -181,7 +256,7 @@ describe('preparation completion and failure', () => { paccount: index === 0 ? 'assets:case' : 'assets:Case' })) }; - queueChildOutput(`${financialPrefix}[${JSON.stringify(original)}]\n[]\n`); + queueChildOutput(`[${JSON.stringify(original)}]\n[]\n`); const transactions = new MemoryPreparationTransactions(); const assertComplete = vi.spyOn(transactions, 'assertComplete'); const writeRows = vi.fn(async () => {}); @@ -196,12 +271,12 @@ describe('preparation completion and failure', () => { }); it('rejects a late native nonzero exit even after row writes', async () => { - const writes = await expectRejected(accountProtocol(51), 9); + const writes = await expectRejected(accountFixture(51), 9); expect(writes).toHaveBeenCalled(); }); it('kills the child and withholds private diagnostics when a row callback fails', async () => { - const child = mockPreparationProcesses(accountProtocol(51), { keepOpen: true }); + const child = mockPreparationProcesses(accountFixture(51), { keepOpen: true }); const error = await prepareReports('/private', { transactions: new MemoryPreparationTransactions(), async writeRows() { @@ -217,7 +292,7 @@ describe('preparation completion and failure', () => { it.each(['original import', 'account rows'] as const)( 'kills the child on abort but waits for the active %s write', async (phase) => { - const child = mockPreparationProcesses(accountProtocol(50), { keepOpen: true }); + const child = mockPreparationProcesses(accountFixture(50), { keepOpen: true }); const childClosed = new Promise((resolve) => { child.once('close', () => resolve()); }); @@ -276,8 +351,11 @@ describe('preparation completion and failure', () => { } ); - it('kills a blocked financial read when aborted', async () => { - const child = mockPreparationProcesses(globalExports.slice(0, -5), { keepOpen: true }); + it('kills a blocked global export read when aborted', async () => { + const child = mockPreparationProcesses( + { output: globalExports.slice(0, -5), count: 1 }, + { keepOpen: true } + ); const controller = new AbortController(); const result = prepareReports('/private', { transactions: new MemoryPreparationTransactions(), @@ -313,25 +391,10 @@ describe('preparation completion and failure', () => { }); }); -describe('financial framing and metadata qualification', () => { - it('does not confuse financial stats-path markers with protocol boundaries', async () => { - mockPreparationProcesses( - `Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${globalExports}[]` - ); - const result = await prepareReports('/private', { - transactions: new MemoryPreparationTransactions(), - async writeRows() {} - }); - expect(result.accounts[0].rowCount).toBe(0); - }); - - it('rejects a forged complete protocol in a stats path unless physical EOF also validates', async () => { - await expectRejected(`${globalExports}[]rest-of-path\n${globalExports}[]`); - }); - +describe('metadata qualification', () => { it('rejects unsafe account identities before starting financial execution', async () => { queueChildOutput(versionOutput); - queueChildOutput(journalStats); + queueChildOutput(emptyStats); queueChildOutput('assets:\tSECRET\n'); const error = await prepareReports('/private/SECRET', { transactions: new MemoryPreparationTransactions(), diff --git a/packages/hledger/src/prepare-reports/run.ts b/packages/hledger/src/prepare-reports/run.ts index 329bb56..2a16ee0 100644 --- a/packages/hledger/src/prepare-reports/run.ts +++ b/packages/hledger/src/prepare-reports/run.ts @@ -6,9 +6,8 @@ import { readPreparationMetadata } from './metadata.js'; import type { PreparationTransactions } from '../transactions/store.js'; import { writePreparedAccountRows, type AccountRowWriter } from './account-stream.js'; import { - decodePreparedPrefix, + readPreparedReports, planPreparedReportCommands, - financialEndMarker, type PreparedAccountSummary } from './batch.js'; @@ -33,9 +32,6 @@ export async function prepareReports( { signal: options.signal }, async (output, check) => { const reader = new ReportStreamReader(output, check); - const { overview, accounts } = await reader.prefix(financialEndMarker, (prefix) => - decodePreparedPrefix(prefix, metadata, overviewPlan) - ); await reader.array('object', async (json) => { await options.transactions.insertOriginal(json); }); @@ -45,6 +41,27 @@ export async function prepareReports( check(); await options.transactions.assertComplete(); check(); + const summary = await options.transactions.readSummary(); + check(); + // Native report spans can include posting dates; compare the latest primary date instead. + if ( + summary.transactionCount !== metadata.journal.journalCoverage.transactionCount || + summary.lastDate !== metadata.journal.asOf + ) + throw new Error('Native transactions do not match preparation metadata.'); + const transactionCounts: number[] = []; + for (const range of overviewPlan.incomeRanges) { + transactionCounts.push( + await options.transactions.countInPeriod(range.begin, range.endExclusive) + ); + check(); + } + const { overview, accounts } = await readPreparedReports( + reader, + metadata, + overviewPlan, + transactionCounts + ); for (const account of accounts) { await writePreparedAccountRows( reader, diff --git a/packages/hledger/src/prepare-reports/transactions.fixture.ts b/packages/hledger/src/prepare-reports/transactions.fixture.ts index 7f00556..d1a1075 100644 --- a/packages/hledger/src/prepare-reports/transactions.fixture.ts +++ b/packages/hledger/src/prepare-reports/transactions.fixture.ts @@ -1,5 +1,9 @@ import { decodeNativeTransaction } from '../transactions/export.js'; -import type { NativeTransactionPair, PreparationTransactions } from '../transactions/store.js'; +import type { + NativeTransactionPair, + NativeTransactionSummary, + PreparationTransactions +} from '../transactions/store.js'; import { assertTransactionExportsMatch, requireTransactionIndex @@ -30,6 +34,24 @@ export class MemoryPreparationTransactions implements PreparationTransactions { } } + async readSummary(): Promise { + let lastDate: string | null = null; + for (const record of this.records.values()) { + const { tdate } = decodeNativeTransaction(record.originalJson); + if (lastDate === null || tdate > lastDate) lastDate = tdate; + } + return { transactionCount: this.records.size, lastDate }; + } + + async countInPeriod(begin: string, endExclusive: string): Promise { + let count = 0; + for (const record of this.records.values()) { + const { tdate } = decodeNativeTransaction(record.originalJson); + if (tdate >= begin && tdate < endExclusive) count += 1; + } + return count; + } + async read(index: number): Promise { requireTransactionIndex(index); const record = this.records.get(index); diff --git a/packages/hledger/src/report-stream.test.ts b/packages/hledger/src/report-stream.test.ts index b2613b3..3c1bf0a 100644 --- a/packages/hledger/src/report-stream.test.ts +++ b/packages/hledger/src/report-stream.test.ts @@ -12,28 +12,37 @@ function reader(text: string | Buffer, split = 1) { ); } +const transaction = String.raw`{ + "tindex": 1, + "tdescription": "東京 λ \\\" ] }", + "tpostings": [{"paccount": "assets:bank", "pamount": [{ + "acommodity": "USD", + "aquantity": {"decimalMantissa": 900719925474099312345, "decimalPlaces": 2, "floatingPoint": 9.007199254740993e+18} + }]}] +}`; + it.each(['array', 'object'] as const)( - 'reads consecutive native %s arrays across UTF-8 and escaped strings', + 'reads a root object then native %s rows across UTF-8 and escaped strings', async (kind) => { - const value = - kind === 'object' - ? { text: '東京 λ \\" ] }', nested: [{ key: [1, 2] }] } - : [{ text: '東京 λ \\" ] }' }]; - const input = reader( - JSON.stringify([value, value]) + '\n' + JSON.stringify([value]) + ' \t\r\n' - ); - const rows: unknown[] = []; + const value = kind === 'object' ? transaction : `[${transaction}]`; + const input = reader(` \t\r\n${transaction}\n[${value}, ${value}] \t\r\n`); + await expect(input.json()).resolves.toBe(transaction); + const rows: string[] = []; await input.array(kind, async (json) => { - rows.push(JSON.parse(json)); + rows.push(json); }); - await input.array(kind, async (json) => { - rows.push(JSON.parse(json)); - }); - expect(rows).toEqual([value, value, value]); + expect(rows).toEqual([value, value]); await input.eof(); } ); +it('preserves a complete native JSON array without changing numeric tokens or internal whitespace', async () => { + const document = `[ ${transaction},\n${transaction} ]`; + const input = reader(`\n${document}\r\n`, 7); + await expect(input.json()).resolves.toBe(document); + await input.eof(); +}); + it.each([ ['array', '[{}]'], ['object', '[[]]'], @@ -50,26 +59,75 @@ it.each([ ).rejects.toThrow(); }); -it('awaits each object callback and propagates failure before consuming the next record', async () => { - const input = reader('[{},{}]', 100); - let release!: () => void; - let entered!: () => void; - const started = new Promise((resolve) => { - entered = resolve; - }); - const held = new Promise((resolve) => { - release = resolve; - }); +it.each(['null', '{"nested":[{}]', '{"text":"unfinished\\'])( + 'rejects a non-container root or truncated native JSON: %s', + async (text) => { + await expect(reader(text).json()).rejects.toThrow(); + } +); + +it.each(['[{}', '[{},{}]'])( + 'awaits the callback and propagates failure before reading another row: %s', + async (first) => { + let pulls = 0; + const input = new ReportStreamReader( + (async function* () { + pulls++; + yield Buffer.from(first); + pulls++; + yield Buffer.from(',{}]'); + })(), + () => {} + ); + let release!: () => void; + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const held = new Promise((resolve) => { + release = resolve; + }); + const callback = vi.fn(async () => { + entered(); + await held; + throw new Error('write failed'); + }); + const result = input.array('object', callback); + await started; + expect(callback).toHaveBeenCalledTimes(1); + expect(pulls).toBe(1); + release(); + await expect(result).rejects.toThrow('write failed'); + expect(callback).toHaveBeenCalledTimes(1); + expect(pulls).toBe(1); + } +); + +it('checks cancellation after input and callback awaits', async () => { + let aborted = false; + const check = () => { + if (aborted) throw new Error('aborted'); + }; + const interrupted = new ReportStreamReader( + (async function* () { + aborted = true; + yield Buffer.from('{}'); + })(), + check + ); + await expect(interrupted.json()).rejects.toThrow('aborted'); + + aborted = false; + const input = new ReportStreamReader( + (async function* () { + yield Buffer.from('[{},{}]'); + })(), + check + ); const callback = vi.fn(async () => { - entered(); - await held; - throw new Error('write failed'); + aborted = true; }); - const result = input.array('object', callback); - await started; - expect(callback).toHaveBeenCalledTimes(1); - release(); - await expect(result).rejects.toThrow('write failed'); + await expect(input.array('object', callback)).rejects.toThrow('aborted'); expect(callback).toHaveBeenCalledTimes(1); }); @@ -84,21 +142,21 @@ it('accepts an object larger than the former 4 MiB threshold across chunks', asy await input.eof(); }); -it('accepts a prefix larger than the former 1 MiB threshold and preserves the next frame', async () => { - const prefix = 'x'.repeat(1024 * 1024 + 1); - const input = reader(`${prefix}\nEND\n[{}]`, 64 * 1024); - await expect(input.prefix('\nEND\n', (text) => text)).resolves.toBe(prefix); - const callback = vi.fn(async () => {}); - await input.array('object', callback); - expect(callback).toHaveBeenCalledWith('{}'); - await input.eof(); +it('rejects invalid and incomplete UTF-8 in native JSON and object exports', async () => { + const invalid = Buffer.from([91, 123, 34, 0xff, 34, 58, 49, 125, 93]); + await expect(reader(invalid).json()).rejects.toThrow(); + await expect(reader(invalid).array('object', async () => {})).rejects.toThrow(); + await expect(reader(Buffer.from([123, 34, 0xe6, 0x9d])).json()).rejects.toThrow(); + const input = reader(Buffer.from([123, 125, 0xe6, 0x9d])); + await expect(input.json()).resolves.toBe('{}'); + await expect(input.eof()).rejects.toThrow(); }); -it('retains fatal UTF-8 and exact EOF contracts for object exports', async () => { - await expect( - reader(Buffer.from([91, 123, 34, 0xff, 34, 58, 49, 125, 93])).array('object', async () => {}) - ).rejects.toThrow(); - const input = reader('[{}]extra'); - await input.array('object', async () => {}); - await expect(input.eof()).rejects.toThrow('Extra native output'); -}); +it.each(['extra', '\u00a0'])( + 'rejects non-JSON whitespace or extra output at EOF: %s', + async (suffix) => { + const input = reader(`[{}]${suffix}`); + await input.array('object', async () => {}); + await expect(input.eof()).rejects.toThrow('Extra native output'); + } +); diff --git a/packages/hledger/src/report-stream.ts b/packages/hledger/src/report-stream.ts index 732af45..eadb3ec 100644 --- a/packages/hledger/src/report-stream.ts +++ b/packages/hledger/src/report-stream.ts @@ -1,4 +1,4 @@ -// Private incremental stream reader. Only one native row is assembled at a time. +// Private incremental reader for native JSON documents and array rows. export class ReportStreamReader { private text = ''; private offset = 0; @@ -15,76 +15,41 @@ export class ReportStreamReader { private async more(): Promise { this.check(); - if (this.offset < this.text.length) return true; - if (this.ended) return false; - const next = await this.input.next(); - this.check(); - this.offset = 0; - this.ended = !!next.done; - if (next.done) this.text = this.decoder.decode(); - else { - this.text = this.decoder.decode(next.value, { stream: true }); + while (this.offset >= this.text.length && !this.ended) { + const next = await this.input.next(); + this.check(); + this.offset = 0; + this.ended = !!next.done; + this.text = next.done + ? this.decoder.decode() + : this.decoder.decode(next.value, { stream: true }); } - return this.text.length > 0 || (!this.ended && (await this.more())); + return this.offset < this.text.length; } - // Candidates are validated as a whole prefix. A stats path containing a - // marker is not a boundary merely because its bytes match that marker. - async prefix(marker: string, decode: (text: string) => T): Promise { - let prefix = ''; - let searched = 0; + private async skipWhitespace(): Promise { while (await this.more()) { - const start = prefix.length; - prefix += this.text.slice(this.offset); - this.offset = this.text.length; - let index = prefix.indexOf(marker, searched); - while (index >= 0) { - const end = index + marker.length; - let result: T; - try { - result = decode(prefix.slice(0, index)); - } catch { - index = prefix.indexOf(marker, end); - continue; - } - // Return unconsumed bytes in the current chunk, without retaining prefix. - this.offset = end - start; - return result; + while (this.offset < this.text.length) { + if (!/^[ \t\r\n]$/.test(this.text[this.offset])) return true; + this.offset++; } - searched = Math.max(0, prefix.length - marker.length + 1); } - throw new Error('Missing native prefix.'); + return false; } - async eof(): Promise { - while (await this.more()) { - if (!/^[ \t\r\n]$/.test(this.text[this.offset++])) throw new Error('Extra native output.'); - } - } - - async array(kind: 'array' | 'object', row: (json: string) => Promise): Promise { - const opening = kind === 'array' ? '[' : '{'; - let state: 'open' | 'first' | 'next' | 'comma' | 'row' = 'open'; + // Native parsers validate the contents; framing preserves their exact text. + async json(): Promise { + if (!(await this.skipWhitespace())) throw new Error('Truncated native JSON.'); + if (this.text[this.offset] !== '{' && this.text[this.offset] !== '[') + throw new Error('Invalid native JSON root.'); let depth = 0; let quoted = false; let escaped = false; - let parts: string[] = []; - while (await this.more()) { - let start = this.offset; + const parts: string[] = []; + do { + const start = this.offset; while (this.offset < this.text.length) { const char = this.text[this.offset++]; - if (state !== 'row') { - if (/^[ \t\r\n]$/.test(char)) continue; - if (state === 'open' && char === '[') state = 'first'; - else if ((state === 'first' || state === 'comma') && char === ']') return; - else if (state === 'comma' && char === ',') state = 'next'; - else if ((state === 'first' || state === 'next') && char === opening) { - state = 'row'; - depth = 1; - start = this.offset - 1; - } else throw new Error('Invalid native array.'); - continue; - } if (quoted) { if (escaped) escaped = false; else if (char === '\\') escaped = true; @@ -93,19 +58,39 @@ export class ReportStreamReader { else if (char === '[' || char === '{') depth++; else if (char === ']' || char === '}') depth--; if (depth === 0) { - const part = this.text.slice(start, this.offset); - parts.push(part); - await row(parts.join('')); - this.check(); - parts = []; - state = 'comma'; + parts.push(this.text.slice(start, this.offset)); + return parts.join(''); } } - if (state === 'row') { - const part = this.text.slice(start); - parts.push(part); - } + parts.push(this.text.slice(start)); + } while (await this.more()); + throw new Error('Truncated native JSON.'); + } + + async eof(): Promise { + if (await this.skipWhitespace()) throw new Error('Extra native output.'); + } + + async array(kind: 'array' | 'object', row: (json: string) => Promise): Promise { + if (!(await this.skipWhitespace())) throw new Error('Truncated native array.'); + if (this.text[this.offset++] !== '[') throw new Error('Invalid native array.'); + if (!(await this.skipWhitespace())) throw new Error('Truncated native array.'); + if (this.text[this.offset] === ']') { + this.offset++; + return; + } + const opening = kind === 'array' ? '[' : '{'; + while (true) { + if (this.text[this.offset] !== opening) throw new Error('Invalid native array row.'); + const json = await this.json(); + this.check(); + await row(json); + this.check(); + if (!(await this.skipWhitespace())) throw new Error('Truncated native array.'); + const delimiter = this.text[this.offset++]; + if (delimiter === ']') return; + if (delimiter !== ',') throw new Error('Invalid native array.'); + if (!(await this.skipWhitespace())) throw new Error('Truncated native array.'); } - throw new Error('Truncated native array.'); } } diff --git a/packages/hledger/src/transactions/index.ts b/packages/hledger/src/transactions/index.ts index f6c4da2..7bdb433 100644 --- a/packages/hledger/src/transactions/index.ts +++ b/packages/hledger/src/transactions/index.ts @@ -1,4 +1,8 @@ export { decodeNativeTransaction } from './export.js'; export { assertTransactionExportsMatch, prepareTransaction } from './transaction.js'; export type { NativeTransaction } from './transaction.js'; -export type { NativeTransactionPair, PreparationTransactions } from './store.js'; +export type { + NativeTransactionPair, + NativeTransactionSummary, + PreparationTransactions +} from './store.js'; diff --git a/packages/hledger/src/transactions/store.ts b/packages/hledger/src/transactions/store.ts index c1f15a0..575c086 100644 --- a/packages/hledger/src/transactions/store.ts +++ b/packages/hledger/src/transactions/store.ts @@ -3,11 +3,18 @@ export interface NativeTransactionPair { transactedJson: string; } +export interface NativeTransactionSummary { + transactionCount: number; + lastDate: string | null; +} + /** Caller-owned storage for one explicit preparation; never a journal writer. */ export interface PreparationTransactions { insertOriginal(json: string): Promise; attachTransacted(json: string): Promise; assertComplete(): Promise; + readSummary(): Promise; + countInPeriod(begin: string, endExclusive: string): Promise; read(nativeIndex: number): Promise; accountHistory(accountName: string): AsyncIterable; } diff --git a/packages/hledger/test-fixtures.ts b/packages/hledger/test-fixtures.ts index 9a95064..cdabc21 100644 --- a/packages/hledger/test-fixtures.ts +++ b/packages/hledger/test-fixtures.ts @@ -1,18 +1,35 @@ import { journalMetadata } from './src/journal-stats.js'; -import { decodeOverviewReports, planOverviewReports } from './src/overview/batch.js'; +import { planOverviewReports, type OverviewReports } from './src/overview/batch.js'; +import { parseBalanceSheet } from './src/overview/balance-sheet.js'; +import { parseIncomeStatement } from './src/overview/income-statement.js'; import { organizeOverviewData } from './src/overview/report.js'; import { planIncomeRanges } from './src/overview/ranges.js'; // Compose only pure production parsers/planners/organizers; never execute native jobs. -export function decodeOverviewFixture(discovery: string, batch: string) { - const metadata = journalMetadata(discovery); - return organizeOverviewData( - decodeOverviewReports(batch, metadata, planOverviewReports(metadata)) - ); +export function decodeOverviewFixture( + discoveryStats: string, + reports: { incomeStatements: string[]; snapshots: string[] }, + transactionCounts: readonly number[] = [1, 3, 6, 12] +) { + const metadata = journalMetadata(discoveryStats); + const plan = planOverviewReports(metadata); + const overview: OverviewReports = { + ...metadata, + incomeStatements: plan.incomeRanges.map((range, index) => ({ + ...range, + transactionCount: transactionCounts[index], + report: parseIncomeStatement(reports.incomeStatements[index]) + })), + snapshots: plan.snapshotPeriods.map((period, index) => ({ + ...period, + report: parseBalanceSheet(reports.snapshots[index]) + })) + }; + return organizeOverviewData(overview); } -// Native framing fixtures shared by the dashboard adapter tests only. -export function balanceFrames(asOf = '2026-04-16'): string { +// Owned native JSON documents shared by the dashboard adapter tests only. +export function balanceReports(asOf = '2026-04-16'): string[] { const date = new Date(`${asOf}T00:00:00Z`); date.setUTCDate(1); date.setUTCMonth(date.getUTCMonth() - 11); @@ -31,36 +48,23 @@ export function balanceFrames(asOf = '2026-04-16'): string { ['Liabilities', { prTotals: { prrAmounts: [[]] } }, false] ] }; - return `\n__HLEDGER_NET_WORTH_${index + 1}__\n${JSON.stringify(report)}\n`; - }).join(''); + return JSON.stringify(report); + }); } -// Empty finite statements; callers can replace a consumed field without rebuilding framing. -export function incomeFrames(asOf = '2026-04-16'): string { - return planIncomeRanges(asOf) - .map((range) => { - const totals = () => ({ prrAmounts: range.periods.map(() => []), prrTotal: [] }); - const report = { - cbrDates: range.periods.map(({ begin, endExclusive }) => - [begin, endExclusive].map((contents) => ({ tag: 'Exact', contents })) - ), - cbrTotals: { prrTotal: [] }, - cbrSubreports: [ - ['Revenues', { prTotals: totals(), prRows: [] }, true], - ['Expenses', { prTotals: totals(), prRows: [] }, false] - ] - }; - return `\n__HLEDGER_INCOME_${range.months}__\n${JSON.stringify(report)}\n`; - }) - .join(''); -} - -// Text stats stay together before every JSON frame. Empty filtered spans are bounded. -export function periodStats(asOf = '2026-04-16', counts = [1, 3, 6, 12]): string { - return planIncomeRanges(asOf) - .map( - (range, index) => - `Base currency : USD\nLast txn : ${counts[index] === 0 ? 'none' : asOf}\nTxns : ${counts[index]} (0.0 per day)\nTxns span : ${range.begin} to ${range.endExclusive}\n` - ) - .join(''); +export function incomeReports(asOf = '2026-04-16'): string[] { + return planIncomeRanges(asOf).map((range) => { + const totals = () => ({ prrAmounts: range.periods.map(() => []), prrTotal: [] }); + const report = { + cbrDates: range.periods.map(({ begin, endExclusive }) => + [begin, endExclusive].map((contents) => ({ tag: 'Exact', contents })) + ), + cbrTotals: { prrTotal: [] }, + cbrSubreports: [ + ['Revenues', { prTotals: totals(), prRows: [] }, true], + ['Expenses', { prTotals: totals(), prRows: [] }, false] + ] + }; + return JSON.stringify(report); + }); } diff --git a/packages/snapshots/src/reports/publication.ts b/packages/snapshots/src/reports/publication.ts index b1bdbbb..ad8f22c 100644 --- a/packages/snapshots/src/reports/publication.ts +++ b/packages/snapshots/src/reports/publication.ts @@ -36,6 +36,8 @@ import { attachTransactedTransaction, assertTransactionImportComplete, readNativeTransaction, + readTransactionSummary, + countTransactionsInPeriod, readAccountTransactionBatch, type StoredNativeTransaction } from '@hledger-reports/db/transactions'; @@ -251,6 +253,12 @@ export async function createReportStage( assertComplete() { return operation(() => assertTransactionImportComplete(writer)); }, + readSummary() { + return operation(() => readTransactionSummary(writer)); + }, + countInPeriod(begin, endExclusive) { + return operation(() => countTransactionsInPeriod(writer, begin, endExclusive)); + }, read(nativeIndex) { return operation(async () => nativePair(await readNativeTransaction(writer, nativeIndex))