Prepare shared native transaction exports #50

Merged
eleith merged 1 commits from feat/export-once-native-transactions into main 2026-11-12 15:46:41 +00:00
28 changed files with 772 additions and 432 deletions

No files matched your search

@@ -0,0 +1,5 @@
---
'@hledger-reports/worker': patch
---
Prepare one shared pair of native transaction exports instead of repeating them for every account.
@@ -2,14 +2,17 @@ import { enqueuePreparation } from './test-fixtures.js';
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { afterEach, expect, it } from 'vitest';
import { openJobsClient, prepareReportsSpec, type JobsClient } from '@hledger-reports/jobs';
import { decodeNativeTransaction } from '@hledger-reports/hledger/transactions';
import {
createReportStage,
readAccountPage,
readOverview,
readReportPublication,
readValidation
readValidation,
type ReportPublication
} from '@hledger-reports/snapshots';
import { lotsJournal as source } from '../../../fixtures/journal.js';
import { startWorker } from '../../index.js';
@@ -59,6 +62,71 @@ async function settled(client: JobsClient, status: 'completed' | 'failed' = 'com
return (await preparations(client))[0]!;
}
function expectSavedNativeExports(
publication: ReportPublication,
accountRowCount: number,
includedSource: string
) {
const database = new DatabaseSync(join(publication.directory, 'reports.sqlite'), {
readOnly: true
});
try {
const stored = database
.prepare(
'SELECT native_index, original_json, transacted_json FROM transactions ORDER BY native_index'
)
.all();
expect(stored).toHaveLength(publication.overview.journalCoverage.transactionCount);
expect(stored.length).toBeGreaterThan(accountRowCount);
const pairs = stored.map((record) => {
expect(record.original_json).toEqual(expect.any(String));
expect(record.transacted_json).toEqual(expect.any(String));
const original = decodeNativeTransaction(record.original_json as string);
const transacted = decodeNativeTransaction(record.transacted_json as string);
for (const transaction of [original, transacted]) {
expect(transaction.tindex).toBe(record.native_index);
expect(transaction.tpostings.length).toBeGreaterThan(0);
expect(transaction.tsourcepos).toContainEqual(
expect.objectContaining({ sourceName: includedSource, sourceLine: expect.any(Number) })
);
}
expect(transacted.tdescription).toBe(original.tdescription);
return { original, transacted };
});
const purchase = pairs.find(
({ original }) => original.tdescription === 'Brokerage | Buy VTI and BND'
)!;
expect(purchase.original.tpostings).toEqual(
expect.arrayContaining([
expect.objectContaining({
paccount: 'assets:investments:brokerage',
pamount: expect.arrayContaining([
expect.objectContaining({
acommodity: 'VTI',
acost: expect.objectContaining({
tag: 'UnitCost',
contents: expect.objectContaining({ acommodity: 'USD' })
})
})
])
}),
expect.objectContaining({ paccount: 'assets:bank:checking' })
])
);
const transfer = pairs.find(
({ original }) => original.tdescription === 'Transfer | Move two VTI to retirement'
)!;
for (const transaction of [transfer.original, transfer.transacted]) {
expect(transaction.tpostings.map((posting) => posting.paccount)).toEqual([
'assets:investments:brokerage',
'assets:investments:retirement'
]);
}
} finally {
database.close();
}
}
it('explicitly prepares a saved overview through the real SDK without a browser and reloads with the worker stopped', async () => {
const f = await fixture();
const client = await openJobsClient(f.jobs);
@@ -122,6 +190,7 @@ it('explicitly prepares a saved overview through the real SDK without a browser
limit: 50
});
expect(savedPage).toEqual(page);
expectSavedNativeExports(savedPublication, page.rows.length, f.included);
expect(await readFile(f.journal, 'utf8')).toBe('include included.journal\n');
expect(await readFile(f.included, 'utf8')).toBe(source);
} finally {
@@ -9,7 +9,8 @@ import {
createReportStage,
readReportPublication,
writeValidation,
type ReportPublication
type ReportPublication,
type ReportStage
} from '@hledger-reports/snapshots';
import { createPrepareReports } from './run.js';
@@ -50,10 +51,17 @@ const previous: ReportPublication = {
directory: '/owned/snapshots/.sets/fixture'
};
const stage = {
transactions: {
insertOriginal: vi.fn<ReportStage['transactions']['insertOriginal']>(),
attachTransacted: vi.fn<ReportStage['transactions']['attachTransacted']>(),
assertComplete: vi.fn<ReportStage['transactions']['assertComplete']>(),
read: vi.fn<ReportStage['transactions']['read']>(),
accountHistory: vi.fn<ReportStage['transactions']['accountHistory']>()
},
writeRows: vi.fn(async () => {}),
publish: vi.fn(async () => {}),
discard: vi.fn(async () => {})
};
} satisfies ReportStage;
const prepare = createPrepareReports(journal, directory);
const run = { id: 'example', createdAt: new Date('2026-04-16T00:00:00Z') };
const checkpoint = vi.fn();
@@ -142,6 +150,7 @@ it.each([true, false])(
vi.mocked(createReportStage).mock.invocationCallOrder[0]!
);
expect(prepareReports).toHaveBeenCalledExactlyOnceWith(journal, {
transactions: stage.transactions,
writeRows: stage.writeRows
});
expect(stage.writeRows).toHaveBeenCalledExactlyOnceWith('assets:cash', []);
@@ -32,7 +32,10 @@ export function createPrepareReports(journal: string, directory: string): Prepar
previous?.journal === journal ? previous.accounts : null
);
try {
const reports = await prepareReports(journal, { writeRows: stage.writeRows });
const reports = await prepareReports(journal, {
transactions: stage.transactions,
writeRows: stage.writeRows
});
await stage.publish(reports.overview, reports.accounts, journal);
return reports.overview;
} finally {
@@ -7,6 +7,7 @@ import { expect, it } from 'vitest';
import { lotsJournal } from '../../../../apps/worker/fixtures/journal.js';
import { formatQuantity, parseExactJson, type Amount } from '../amounts.js';
import { prepareReports } from '../prepare-reports/run.js';
import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js';
const exec = promisify(execFile);
@@ -18,7 +19,10 @@ const amounts = (bag: Amount[]) =>
}));
async function compareNativeSubtree(journal: string, name: string) {
const prepared = await prepareReports(journal, { async writeRows() {} });
const prepared = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
async writeRows() {}
});
const { asOf } = prepared.overview;
const base = prepared.overview.netWorth.currency!;
expect(asOf).not.toBeNull();
+1
View File
@@ -16,6 +16,7 @@ export type { ReportedAmount } from './amounts.js';
export { runHoldings } from './holdings/report.js';
export type { HoldingPosition, HoldingsAccount, HoldingsReport } from './holdings/report.js';
export { prepareReports } from './prepare-reports/run.js';
export type { PreparationTransactions } from './transactions/store.js';
export type { PreparedAccountSummary } from './prepare-reports/batch.js';
export type { PreparedBalance } from './accounts/balances.js';
export type { PreparedAccountRow } from './transactions/account-rows.js';
@@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
import { prepareReports, getHledgerVersion } from '../index.js';
import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js';
import {
balanceFrames,
decodeOverviewFixture,
@@ -212,7 +213,10 @@ describe('native market assets minus liabilities snapshots', () => {
txn('2026-03-01', 'Debt', 'Wallet', '100.00 USD') +
txn('2026-04-16', 'Costs', 'Wallet', '1000.00 USD')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth).toEqual({
currency: 'USD',
points: [
@@ -236,7 +240,10 @@ describe('native market assets minus liabilities snapshots', () => {
journal,
'commodity USD 1.00\n' + accounts + txn('2026-04-16', 'Wallet', 'Debt', '100.00 USD')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth?.points.slice(0, 11)).toEqual(
dates.slice(0, 11).map((date) => ({ date, amounts: [], unconverted: [] }))
);
@@ -256,7 +263,10 @@ describe('native market assets minus liabilities snapshots', () => {
txn('2025-01-01', 'Shares', 'Equity', '2 STOCK') +
txn('2026-04-16', 'Wallet', 'Equity', '1.00 USD')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth).toEqual({
currency: 'USD',
points: [...Array<string>(6).fill('40'), '40', '40', '80', '80', '80', '121.00'].map(
@@ -276,7 +286,10 @@ describe('native market assets minus liabilities snapshots', () => {
accounts +
'P 2025-01-01 USD 1.00 USD\n\n2026-04-16 Purchase\n Shares 2 STOCK @ 50.00 USD\n Wallet -100.00 USD\n'
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth?.currency).toBe('USD');
expect(data.netWorth?.points[11]).toEqual({
date: dates[11],
@@ -293,7 +306,10 @@ describe('native market assets minus liabilities snapshots', () => {
'P 2025-01-01 STOCK 10.00 EUR\nP 2099-01-01 EUR 2.00 USD\nP 2099-02-01 EUR 3.00 USD\n' +
txn('2026-04-16', 'Shares', 'Equity', '2 STOCK')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth?.currency).toBe('USD');
expect(data.netWorth?.points[11]).toEqual({
date: dates[11],
@@ -310,7 +326,10 @@ describe('native market assets minus liabilities snapshots', () => {
'P 2025-01-01 USD 1.00 USD\n' +
txn('2026-04-16', 'Shares', 'Debt', '2 STOCK')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth?.currency).toBe('USD');
expect(data.netWorth?.points[11]?.unconverted).toEqual(['STOCK']);
expect(data.netWorth?.points[11]?.amounts).toEqual([q('0', 'STOCK')]);
@@ -330,7 +349,10 @@ describe('native market assets minus liabilities snapshots', () => {
accounts +
txn('2026-04-16', 'Wallet', 'Equity', `42.00 ${symbol}`)
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.netWorth?.currency).toBe(currency);
expect(data.netWorth?.points[11]).toEqual({
date: dates[11],
+5 -1
View File
@@ -6,6 +6,7 @@ import { planIncomeRanges, organizeIncomeRange } from './ranges.js';
import type { HledgerIncomeStatement } from './income-statement.js';
import type { Amount } from '../amounts.js';
import { prepareReports, getHledgerVersion } from '../index.js';
import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js';
const plan = planIncomeRanges('2026-04-16')[3];
function statement(range = plan): HledgerIncomeStatement {
@@ -162,7 +163,10 @@ describe('native hledger 1.99.5 finite overview ranges', () => {
'\n2026-04-12 Later posting\n Costs:Food 700.00 USD ; date:2026-04-17\n Wallet\n' +
'\n~ monthly from 2025-05-01\n Costs:Food 900.00 USD\n Wallet\n';
await writeFile(journal, text);
const { overview } = await prepareReports(journal, { writeRows: async () => {} });
const { overview } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(overview.asOf).toBe('2026-04-16');
for (const [months, begin, income, expenses, net, food] of [
[1, '2026-04-01', '400.00', '35.00', '365.00', '40.00'],
+22 -5
View File
@@ -3,6 +3,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import { prepareReports, getHledgerVersion } from '../index.js';
import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js';
import {
balanceFrames,
decodeOverviewFixture,
@@ -126,7 +127,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
it('keeps twelve empty income columns and four empty ranges with a leap-day asset anchor', async () => {
await writeFile(journal, declarations + posting('2024-02-29', 'Wallet', 'Reserve', '1.00 USD'));
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.asOf).toBe('2024-02-29');
expect(data.incomeExpenses.months).toHaveLength(12);
for (const month of data.incomeExpenses.months) {
@@ -149,7 +153,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
declarations +
'P 2099-12-31 EUR 2.00 USD\n\n~ monthly from 2099-01-01\n Spending 1.00 USD\n Wallet\n'
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data).toEqual({
asOf: null,
journalCoverage: emptyCoverage,
@@ -174,7 +181,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
'\n2040-06-01 Earlier posting date\n Wallet 2.00 USD\n Paycheck -2.00 USD ; date:2039-01-01\n' +
'\n~ monthly from 2099-01-01\n Spending 1.00 USD\n Wallet\n'
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.asOf).toBe('2040-07-12');
expect(data.journalCoverage).toEqual({
transactionCount: 5,
@@ -195,7 +205,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
posting('2026-04-02', 'Spending', 'Wallet', '100.00 USD') +
posting('2026-05-03', 'Wallet', 'Investment gains', '250.00 USD')
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.asOf).toBe('2026-05-03');
expect(data.incomeExpenses.months.find(({ begin }) => begin === '2026-04-01')).toMatchObject({
income: [{ commodity: 'USD', quantity: '1000.00' }],
@@ -216,7 +229,10 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
'\n2026-04-02 Stock expense\n Spending 2 STOCK @ 50.00 USD\n Wallet -100.00 USD\n' +
'\n2026-04-03 Other stock cost\n Spending 3 STOCK @ 60.00 USD\n Wallet -180.00 USD\n'
);
const { overview: data } = await prepareReports(journal, { writeRows: async () => {} });
const { overview: data } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(data.asOf).toBe('2026-04-03');
const bag = [
{ commodity: 'EUR', quantity: '9007199254740993.06' },
@@ -238,6 +254,7 @@ describe('native hledger 1.99.5 journal-anchored income statement', () => {
declarations + posting('2026-04-15', 'SECRET undeclared', 'Wallet', '123.45 USD')
);
const error: unknown = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
}).catch((error: unknown) => error);
expect(error).toBeInstanceOf(Error);
@@ -4,6 +4,7 @@ import { join } from 'node:path';
import { afterAll, beforeAll, expect, it } from 'vitest';
import { transactionCountsJournal } from '../../fixtures/transaction-counts.fixture.js';
import { prepareReports } from '../prepare-reports/run.js';
import { MemoryPreparationTransactions } from '../prepare-reports/transactions.fixture.js';
let directory: string;
let journal: string;
@@ -18,7 +19,10 @@ afterAll(async () => {
const expected = { 1: 2, 3: 4, 6: 6, 12: 8 };
it('prepares all four native counts alongside whole coverage and the existing financial set', async () => {
const { overview } = await prepareReports(journal, { writeRows: async () => {} });
const { overview } = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
writeRows: async () => {}
});
expect(
Object.fromEntries(
Object.entries(overview.ranges!).map(([months, range]) => [months, range.transactionCount])
@@ -4,28 +4,47 @@ import {
organizeAccountRow,
type PreparedAccountRow
} from '../transactions/account-rows.js';
import { AccountTransactionExports } from '../transactions/export.js';
import { decodeNativeTransaction } from '../transactions/export.js';
import { accountInvestmentContext, type InvestmentMetadata } from '../transactions/investment.js';
import { accountMarkers, type PreparedAccountSummary } from './batch.js';
import type { NativeTransactionPair, PreparationTransactions } from '../transactions/store.js';
import { prepareTransaction, requireTransactionIndex } from '../transactions/transaction.js';
import type { PreparedAccountSummary } from './batch.js';
export type AccountRowWriter = (
accountName: string,
rows: readonly PreparedAccountRow[]
) => Promise<void>;
function prepareAccountTransaction(pair: NativeTransactionPair, accountName: string) {
const original = decodeNativeTransaction(pair.originalJson);
const transaction = prepareTransaction(original, decodeNativeTransaction(pair.transactedJson));
if (
!transaction.postings.some(
(posting) => posting.account === accountName || posting.account.startsWith(`${accountName}:`)
)
) {
throw new Error('Native transaction does not belong to the account subtree.');
}
return transaction;
}
export async function writePreparedAccountRows(
reader: ReportStreamReader,
account: PreparedAccountSummary,
metadata: InvestmentMetadata,
transactions: PreparationTransactions,
writeRows: AccountRowWriter,
check: () => void
): Promise<void> {
const exports = new AccountTransactionExports(account.name, account.access === 'ambiguous');
await reader.array('object', async (json) => exports.original(json));
await reader.expect(`\n${accountMarkers.originalEnd}`);
await reader.array('object', async (json) => exports.transacted(json));
await reader.expect(`\n${accountMarkers.transactedEnd}`);
const context = accountInvestmentContext(exports.transactions(), metadata, account.name);
if (account.access === 'ambiguous') return;
async function* history() {
for await (const pair of transactions.accountHistory(account.name)) {
check();
yield prepareAccountTransaction(pair, account.name);
}
}
const context = await accountInvestmentContext(history(), metadata, account.name);
const indexes = new Set<number>();
let chunk: PreparedAccountRow[] = [];
const write = async () => {
check();
@@ -36,13 +55,18 @@ export async function writePreparedAccountRows(
chunk = [];
};
await reader.array('array', async (json) => {
check();
const tuple = parseAccountRegisterRow(json);
if (account.access === 'ambiguous') return;
const transaction = exports.take(tuple[0].tindex);
const index = tuple[0].tindex;
requireTransactionIndex(index);
if (indexes.has(index)) throw new Error('Missing or duplicate native register index.');
indexes.add(index);
const pair = await transactions.read(index);
check();
const transaction = prepareAccountTransaction(pair, account.name);
chunk.push(organizeAccountRow(tuple, account.name, transaction, context));
account.rowCount = (account.rowCount ?? 0) + 1;
if (chunk.length === 50) await write();
});
await reader.expect(`\n${accountMarkers.registerEnd}`);
if (account.access === 'ready' && (chunk.length > 0 || account.rowCount === 0)) await write();
if (chunk.length > 0 || account.rowCount === 0) await write();
}
@@ -0,0 +1,37 @@
import { expect, it } from 'vitest';
import { journalMetadata } from '../journal-stats.js';
import { planPreparedReportCommands } from './batch.js';
it('exports the whole journal once regardless of account count or ambiguous selections', () => {
const { args } = planPreparedReportCommands([], {
journal: journalMetadata('Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n'),
accounts: [
{ name: 'assets:cash', access: 'ready' },
{ name: 'assets:cash:daily', access: 'ready' },
{ name: 'assets:Case', access: 'ambiguous' }
],
investment: {
assets: new Set(),
cash: new Set(),
liabilities: new Set(),
income: new Set(),
expenses: new Set(),
gains: new Set(),
lotCommodities: new Set()
}
});
expect(args.filter((argument) => argument === 'print')).toHaveLength(2);
expect(args.slice(args.indexOf('print'), args.indexOf('aregister'))).toEqual([
'print',
'--round=none',
'-Ojson',
'--',
'print',
'--round=none',
'--value=transacted',
'-Ojson',
'--'
]);
expect(args.filter((argument) => argument === 'aregister')).toHaveLength(2);
expect(args).not.toContain('^assets:Case$');
});
+16 -39
View File
@@ -13,12 +13,8 @@ export interface PreparedAccountSummary extends QualifiedAccount {
summary: PreparedBalance;
}
export const registersMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n';
export const accountMarkers = {
originalEnd: '\n__HLEDGER_PREPARED_ORIGINAL_END__\n',
transactedEnd: '\n__HLEDGER_PREPARED_TRANSACTED_END__\n',
registerEnd: '\n__HLEDGER_PREPARED_REGISTER_END__\n'
} as const;
// Keep the legacy delimiter for the finite financial prefix.
export const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n';
function echo(marker: string): string[] {
return ['--', 'echo', marker.trimEnd()];
@@ -39,41 +35,22 @@ export function planPreparedReportCommands(
'--',
...overviewPlan.commands,
...planAccountBalances(metadata.journal),
...echo(registersMarker)
...echo(financialEndMarker),
'--',
'print',
'--round=none',
'-Ojson',
'--',
'print',
'--round=none',
'--value=transacted',
'-Ojson'
];
for (const { name } of metadata.accounts) {
const escapedName = escapeAccountName(name);
const exactAccountPattern = `^${escapedName}$`;
// Print's acct query is case-insensitive; exact canonical membership is
// checked after decoding, without dropping any counterpart postings.
const subtreeQuery = `acct:^${escapedName}(:|$)`;
args.push(
'--',
'print',
'--round=none',
'-Ojson',
subtreeQuery,
...echo(accountMarkers.originalEnd)
);
args.push(
'--',
'print',
'--round=none',
'--value=transacted',
'-Ojson',
subtreeQuery,
...echo(accountMarkers.transactedEnd)
);
for (const { name, access } of metadata.accounts) {
if (access === 'ambiguous') continue;
const exactAccountPattern = `^${escapeAccountName(name)}$`;
// No separator before the literal pattern: run owns `--` separators.
args.push(
'--',
'aregister',
'--historical',
'--empty',
'-Ojson',
exactAccountPattern,
...echo(accountMarkers.registerEnd)
);
args.push('--', 'aregister', '--historical', '--empty', '-Ojson', exactAccountPattern);
}
return { args, overviewPlan };
}
@@ -4,7 +4,10 @@ import { join } from 'node:path';
import { afterEach, expect, it } from 'vitest';
import { cashJournal } from '../../fixtures/cash.fixture.js';
import type { PreparedAccountRow } from '../transactions/account-rows.js';
import { decodeNativeTransaction } from '../transactions/export.js';
import { prepareTransaction } from '../transactions/transaction.js';
import { prepareReports } from './run.js';
import { MemoryPreparationTransactions } from './transactions.fixture.js';
const directories: string[] = [];
@@ -26,6 +29,7 @@ function collectRows() {
const rowsByAccount = new Map<string, PreparedAccountRow[]>();
return {
rowsByAccount,
transactions: new MemoryPreparationTransactions(),
async writeRows(name: string, rows: readonly PreparedAccountRow[]) {
const previous = rowsByAccount.get(name) ?? [];
previous.push(...rows);
@@ -94,6 +98,7 @@ include detail.journal
const during = collectRows();
let changed = false;
const reports = await prepareReports(journal, {
transactions: during.transactions,
async writeRows(name, rows) {
await during.writeRows(name, rows);
if (!changed) {
@@ -108,7 +113,10 @@ include detail.journal
...beforeReports.overview,
preparedAt: ''
});
const after = await prepareReports(journal, { async writeRows() {} });
const after = await prepareReports(journal, {
transactions: new MemoryPreparationTransactions(),
async writeRows() {}
});
expect(after.overview.netWorth).not.toEqual(reports.overview.netWorth);
});
@@ -121,6 +129,10 @@ account equity
assets:case 1 USD
assets:Case 2 USD
equity
2026-01-02 Only ambiguous accounts
assets:case 3 USD
assets:Case -3 USD
`);
const collected = collectRows();
const prepared = await prepareReports(journal, collected);
@@ -130,4 +142,22 @@ account equity
expect(account).toMatchObject({ access: 'ambiguous', rowCount: null });
expect(collected.rowsByAccount.has(account.name)).toBe(false);
}
expect(collected.transactions.records.size).toBe(2);
const pairs = await Promise.all(
[...collected.transactions.records.keys()].map((index) => collected.transactions.read(index))
);
const transactions = pairs.map(({ originalJson, transactedJson }) =>
prepareTransaction(
decodeNativeTransaction(originalJson),
decodeNativeTransaction(transactedJson)
)
);
expect(transactions.map(({ description }) => description)).toEqual([
'Collision',
'Only ambiguous accounts'
]);
expect(transactions[1].postings).toMatchObject([
{ account: 'assets:case', transactedAmounts: [{ commodity: 'USD', quantity: '3' }] },
{ account: 'assets:Case', transactedAmounts: [{ commodity: 'USD', quantity: '-3' }] }
]);
});
+116 -65
View File
@@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { supportedHledgerVersion } from '../checks.js';
import { transactionFixture } from '../transactions/transaction.fixture.js';
import { prepareReports } from './run.js';
import { MemoryPreparationTransactions } from './transactions.fixture.js';
const processMock = vi.hoisted(() => ({ spawn: vi.fn() }));
vi.mock('node:child_process', async (importOriginal) => {
@@ -35,16 +36,14 @@ afterEach(async () => {
const versionOutput = `hledger ${supportedHledgerVersion}, SECRET banner\n`;
const journalStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n';
const financialPrefix = `${journalStats}\n__HLEDGER_PREPARED_REGISTERS__\n`;
const originalEnd = '\n\n__HLEDGER_PREPARED_ORIGINAL_END__\n';
const transactedEnd = '\n\n__HLEDGER_PREPARED_TRANSACTED_END__\n';
const registerEnd = '\n\n__HLEDGER_PREPARED_REGISTER_END__\n';
const financialEndMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n';
const financialPrefix = `${journalStats}${financialEndMarker}`;
const nativeTransaction = {
...transactionFixture(),
tdate: '2031-01-01',
tdescription: 'λ, test'
};
const accountExports = `${financialPrefix}[${JSON.stringify(nativeTransaction)}]${originalEnd}[${JSON.stringify(nativeTransaction)}]${transactedEnd}`;
const globalExports = `${financialPrefix}[${JSON.stringify(nativeTransaction)}]\n[${JSON.stringify(nativeTransaction)}]\n`;
const registerRow = JSON.stringify([
nativeTransaction,
nativeTransaction,
@@ -67,7 +66,7 @@ function accountProtocol(rowCount: number): string {
const rows = JSON.stringify(
transactions.map((transaction) => [transaction, transaction, false, ['other, 東京'], [], []])
);
return `${financialPrefix}${prints}${originalEnd}${prints}${transactedEnd}${rows}${registerEnd}`;
return `${financialPrefix}${prints}\n${prints}\n${rows}`;
}
function startFixtureChild(script: string): ChildProcessWithoutNullStreams {
@@ -114,9 +113,10 @@ function mockPreparationProcesses(
async function expectRejected(output: string, exitCode = 0) {
const child = mockPreparationProcesses(output, { exitCode });
const writeRows = vi.fn(async () => {});
const error = await prepareReports('/private/SECRET', { writeRows }).catch(
(error: unknown) => error
);
const error = await prepareReports('/private/SECRET', {
transactions: new MemoryPreparationTransactions(),
writeRows
}).catch((error: unknown) => error);
expect(error).toBeInstanceOf(Error);
expect((error as Error).message).not.toMatch(/SECRET|\/private/);
expect((error as Error).cause).toBeUndefined();
@@ -129,8 +129,9 @@ describe('preparation completion and failure', () => {
vi.useFakeTimers({ toFake: ['Date'] });
try {
vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));
mockPreparationProcesses(`${accountExports}[${registerRow}]${registerEnd}`);
mockPreparationProcesses(`${globalExports}[${registerRow}] \t\r\n`);
const result = await prepareReports('/private', {
transactions: new MemoryPreparationTransactions(),
async writeRows() {
vi.setSystemTime(new Date('2026-01-01T00:02:30Z'));
}
@@ -152,18 +153,48 @@ describe('preparation completion and failure', () => {
});
it.each([
{ name: 'truncated account output', output: `${accountExports}[${registerRow}` },
{ name: 'truncated account output', output: `${globalExports}[${registerRow}` },
{
name: 'crossed export frames',
output: `${accountExports}[${registerRow}]${registerEnd}`.replace(
'__HLEDGER_PREPARED_ORIGINAL_END__',
'__WRONG_END__'
)
name: 'malformed native JSON between global exports',
output: `${financialPrefix}[${JSON.stringify(nativeTransaction)}]invalid[]`
}
])('rejects $name without returning a partial report', async ({ output }) => {
await expectRejected(output);
});
it('rejects a repeated register transaction even though shared storage permits rereads', async () => {
const writes = await expectRejected(`${globalExports}[${registerRow},${registerRow}]`);
expect(writes).not.toHaveBeenCalled();
});
it('rejects incomplete global pairs even when every account is ambiguous', async () => {
queueChildOutput(versionOutput);
queueChildOutput(journalStats);
queueChildOutput('assets:case\nassets:Case\n');
queueChildOutput('assets:case ; type: C\nassets:Case ; type: C\n');
queueChildOutput('');
queueChildOutput('assets:case\nassets:case\n');
const original = {
...nativeTransaction,
tpostings: nativeTransaction.tpostings.map((posting, index) => ({
...posting,
paccount: index === 0 ? 'assets:case' : 'assets:Case'
}))
};
queueChildOutput(`${financialPrefix}[${JSON.stringify(original)}]\n[]\n`);
const transactions = new MemoryPreparationTransactions();
const assertComplete = vi.spyOn(transactions, 'assertComplete');
const writeRows = vi.fn(async () => {});
await expect(
prepareReports('/private', {
transactions,
writeRows
})
).rejects.toThrow('Prepared transactions could not be read. Check the journal.');
expect(assertComplete).toHaveBeenCalledOnce();
expect(writeRows).not.toHaveBeenCalled();
});
it('rejects a late native nonzero exit even after row writes', async () => {
const writes = await expectRejected(accountProtocol(51), 9);
expect(writes).toHaveBeenCalled();
@@ -172,6 +203,7 @@ describe('preparation completion and failure', () => {
it('kills the child and withholds private diagnostics when a row callback fails', async () => {
const child = mockPreparationProcesses(accountProtocol(51), { keepOpen: true });
const error = await prepareReports('/private', {
transactions: new MemoryPreparationTransactions(),
async writeRows() {
throw new Error('SECRET /private 123.45 USD');
}
@@ -182,57 +214,73 @@ describe('preparation completion and failure', () => {
expect(child.signalCode).toBe('SIGKILL');
});
it('kills the child on abort but waits for the active row write before returning', async () => {
const child = mockPreparationProcesses(accountProtocol(50), { keepOpen: true });
const childClosed = new Promise<void>((resolve) => {
child.once('close', () => resolve());
});
const controller = new AbortController();
let enterWrite!: () => void;
const firstWrite = new Promise<void>((resolve) => {
enterWrite = resolve;
});
let releaseWrite!: () => void;
const held = new Promise<void>((resolve) => {
releaseWrite = resolve;
});
releaseWrites.push(releaseWrite);
let writeFinished = false;
let settled = false;
const result = prepareReports('/private', {
signal: controller.signal,
async writeRows() {
it.each(['original import', 'account rows'] as const)(
'kills the child on abort but waits for the active %s write',
async (phase) => {
const child = mockPreparationProcesses(accountProtocol(50), { keepOpen: true });
const childClosed = new Promise<void>((resolve) => {
child.once('close', () => resolve());
});
const controller = new AbortController();
let enterWrite!: () => void;
const firstWrite = new Promise<void>((resolve) => {
enterWrite = resolve;
});
let releaseWrite!: () => void;
const held = new Promise<void>((resolve) => {
releaseWrite = resolve;
});
releaseWrites.push(releaseWrite);
let writeFinished = false;
let settled = false;
const transactions = new MemoryPreparationTransactions();
const write = async () => {
enterWrite();
await held;
writeFinished = true;
};
if (phase === 'original import') {
const insertOriginal = transactions.insertOriginal.bind(transactions);
vi.spyOn(transactions, 'insertOriginal').mockImplementationOnce(async (json) => {
await write();
await insertOriginal(json);
});
}
});
void result.then(
() => {
settled = true;
},
() => {
settled = true;
const result = prepareReports('/private', {
transactions,
signal: controller.signal,
async writeRows() {
if (phase === 'account rows') await write();
}
});
void result.then(
() => {
settled = true;
},
() => {
settled = true;
}
);
try {
await firstWrite;
controller.abort();
await childClosed;
expect(child.signalCode).toBe('SIGKILL');
expect(settled).toBe(false);
expect(writeFinished).toBe(false);
} finally {
releaseWrite();
}
);
try {
await firstWrite;
controller.abort();
await childClosed;
expect(child.signalCode).toBe('SIGKILL');
expect(settled).toBe(false);
expect(writeFinished).toBe(false);
} finally {
releaseWrite();
await expect(result).rejects.toThrow();
expect(writeFinished).toBe(true);
}
await expect(result).rejects.toThrow();
expect(writeFinished).toBe(true);
});
);
it('kills a blocked financial read when aborted', async () => {
const child = mockPreparationProcesses(accountExports.slice(0, -5), { keepOpen: true });
const child = mockPreparationProcesses(globalExports.slice(0, -5), { keepOpen: true });
const controller = new AbortController();
const result = prepareReports('/private', {
transactions: new MemoryPreparationTransactions(),
signal: controller.signal,
async writeRows() {}
});
@@ -252,6 +300,7 @@ describe('preparation completion and failure', () => {
});
const controller = new AbortController();
const result = prepareReports('/private', {
transactions: new MemoryPreparationTransactions(),
signal: controller.signal,
async writeRows() {}
});
@@ -267,25 +316,27 @@ describe('preparation completion and failure', () => {
describe('financial framing and metadata qualification', () => {
it('does not confuse financial stats-path markers with protocol boundaries', async () => {
mockPreparationProcesses(
`Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${accountExports}[]${registerEnd}`
`Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${globalExports}[]`
);
const result = await prepareReports('/private', { async writeRows() {} });
const result = await prepareReports('/private', {
transactions: new MemoryPreparationTransactions(),
async writeRows() {}
});
expect(result.accounts[0].rowCount).toBe(0);
});
it('rejects a forged complete protocol in a stats path unless physical EOF also validates', async () => {
await expectRejected(
`${accountExports}[]${registerEnd}rest-of-path\n${accountExports}[]${registerEnd}`
);
await expectRejected(`${globalExports}[]rest-of-path\n${globalExports}[]`);
});
it('rejects unsafe account identities before starting financial execution', async () => {
queueChildOutput(versionOutput);
queueChildOutput(journalStats);
queueChildOutput('assets:\tSECRET\n');
const error = await prepareReports('/private/SECRET', { async writeRows() {} }).catch(
(error: unknown) => error
);
const error = await prepareReports('/private/SECRET', {
transactions: new MemoryPreparationTransactions(),
async writeRows() {}
}).catch((error: unknown) => error);
expect(error).toBeInstanceOf(Error);
expect((error as Error).message).not.toMatch(/SECRET|\/private/);
expect((error as Error).cause).toBeUndefined();
+14 -2
View File
@@ -3,17 +3,19 @@ import type { OverviewData } from '../overview/report.js';
import { consumeHledger, readHledger } from '../process.js';
import { ReportStreamReader } from '../report-stream.js';
import { readPreparationMetadata } from './metadata.js';
import type { PreparationTransactions } from '../transactions/store.js';
import { writePreparedAccountRows, type AccountRowWriter } from './account-stream.js';
import {
decodePreparedPrefix,
planPreparedReportCommands,
registersMarker,
financialEndMarker,
type PreparedAccountSummary
} from './batch.js';
export async function prepareReports(
journal: string,
options: {
transactions: PreparationTransactions;
writeRows: AccountRowWriter;
signal?: AbortSignal;
}
@@ -31,14 +33,24 @@ export async function prepareReports(
{ signal: options.signal },
async (output, check) => {
const reader = new ReportStreamReader(output, check);
const { overview, accounts } = await reader.prefix(registersMarker, (prefix) =>
const { overview, accounts } = await reader.prefix(financialEndMarker, (prefix) =>
decodePreparedPrefix(prefix, metadata, overviewPlan)
);
await reader.array('object', async (json) => {
await options.transactions.insertOriginal(json);
});
await reader.array('object', async (json) => {
await options.transactions.attachTransacted(json);
});
check();
await options.transactions.assertComplete();
check();
for (const account of accounts) {
await writePreparedAccountRows(
reader,
account,
metadata.investment,
options.transactions,
options.writeRows,
check
);
@@ -0,0 +1,55 @@
import { decodeNativeTransaction } from '../transactions/export.js';
import type { NativeTransactionPair, PreparationTransactions } from '../transactions/store.js';
import {
assertTransactionExportsMatch,
requireTransactionIndex
} from '../transactions/transaction.js';
/** Owned, in-memory storage for SDK preparation tests only. */
export class MemoryPreparationTransactions implements PreparationTransactions {
readonly records = new Map<number, { originalJson: string; transactedJson: string | null }>();
async insertOriginal(json: string): Promise<void> {
const transaction = decodeNativeTransaction(json);
if (this.records.has(transaction.tindex)) throw new Error('Duplicate native original index.');
this.records.set(transaction.tindex, { originalJson: json, transactedJson: null });
}
async attachTransacted(json: string): Promise<void> {
const transaction = decodeNativeTransaction(json);
const record = this.records.get(transaction.tindex);
if (!record) throw new Error('Missing native original index.');
if (record.transactedJson !== null) throw new Error('Duplicate native transacted index.');
assertTransactionExportsMatch(decodeNativeTransaction(record.originalJson), transaction);
record.transactedJson = json;
}
async assertComplete(): Promise<void> {
for (const record of this.records.values()) {
if (record.transactedJson === null) throw new Error('Missing native transacted index.');
}
}
async read(index: number): Promise<NativeTransactionPair> {
requireTransactionIndex(index);
const record = this.records.get(index);
if (!record || record.transactedJson === null)
throw new Error('Missing native transaction pair.');
return { originalJson: record.originalJson, transactedJson: record.transactedJson };
}
async *accountHistory(accountName: string): AsyncIterable<NativeTransactionPair> {
for (const index of [...this.records.keys()].sort((left, right) => left - right)) {
const pair = await this.read(index);
const transaction = decodeNativeTransaction(pair.originalJson);
if (
transaction.tpostings.some(({ paccount }) => {
if (typeof paccount !== 'string') throw new Error('Invalid native posting account.');
return paccount === accountName || paccount.startsWith(`${accountName}:`);
})
) {
yield pair;
}
}
}
}
+8 -4
View File
@@ -13,19 +13,23 @@ function reader(text: string | Buffer, split = 1) {
}
it.each(['array', 'object'] as const)(
'frames %s records separately across UTF-8 and escaped string boundaries',
'reads consecutive native %s arrays across UTF-8 and escaped strings',
async (kind) => {
const value =
kind === 'object'
? { text: '東京 λ \\" ] }', nested: [{ key: [1, 2] }] }
: [{ text: '東京 λ \\" ] }' }];
const input = reader(JSON.stringify([value, value]) + '\nEND');
const input = reader(
JSON.stringify([value, value]) + '\n' + JSON.stringify([value]) + ' \t\r\n'
);
const rows: unknown[] = [];
await input.array(kind, async (json) => {
rows.push(JSON.parse(json));
});
expect(rows).toEqual([value, value]);
await input.expect('\nEND');
await input.array(kind, async (json) => {
rows.push(JSON.parse(json));
});
expect(rows).toEqual([value, value, value]);
await input.eof();
}
);
+3 -9
View File
@@ -56,16 +56,10 @@ export class ReportStreamReader {
throw new Error('Missing native prefix.');
}
async expect(literal: string): Promise<void> {
for (const char of literal) {
if (!(await this.more()) || this.text[this.offset++] !== char) {
throw new Error('Invalid native separator.');
}
}
}
async eof(): Promise<void> {
if (await this.more()) throw new Error('Extra native output.');
while (await this.more()) {
if (!/^[ \t\r\n]$/.test(this.text[this.offset++])) throw new Error('Extra native output.');
}
}
async array(kind: 'array' | 'object', row: (json: string) => Promise<void>): Promise<void> {
@@ -13,13 +13,13 @@ const metadata: InvestmentMetadata = {
gains: new Set(),
lotCommodities: new Set()
};
function organize(input: ReturnType<typeof tuple>, account: string) {
async function organize(input: ReturnType<typeof tuple>, account: string) {
const transaction = prepareTransaction(input[0], structuredClone(input[0]));
return organizeAccountRow(
input,
account,
transaction,
accountInvestmentContext([transaction], metadata, account)
await accountInvestmentContext([transaction], metadata, account)
);
}
@@ -37,11 +37,11 @@ function tuple() {
);
}
it('retains original metadata alongside account-context date and exact reported bags', () => {
it('retains original metadata alongside account-context date and exact reported bags', async () => {
const input = tuple();
input[4][0].aquantity.decimalMantissa = -900719925474099301n;
const before = structuredClone(input);
const result = organize(input, 'assets:cash');
const result = await organize(input, 'assets:cash');
expect(result).toMatchObject({
accountDate: '2026-04-16',
isSplit: true,
@@ -61,7 +61,7 @@ it('retains original metadata alongside account-context date and exact reported
expect(JSON.parse(JSON.stringify(result))).toEqual(result);
});
it('matches qualified canonical account/subtree case-sensitively, independent of OtherAccounts', () => {
it('matches qualified canonical account/subtree case-sensitively, independent of OtherAccounts', async () => {
const original = transactionFixture();
const names = [
'assets:cash',
@@ -79,10 +79,10 @@ it('matches qualified canonical account/subtree case-sensitively, independent of
const input = parseAccountRegisterRow(
JSON.stringify([original, original, false, ['assets:cash', 'assets:cash:Child'], [], []])
);
expect(organize(input, 'assets:cash').accountPostingIndexes).toEqual([0, 1, 5]);
expect((await organize(input, 'assets:cash')).accountPostingIndexes).toEqual([0, 1, 5]);
});
it('preserves ordered duplicate commodities, mixed units, unnamed and explicit zero entries', () => {
it('preserves ordered duplicate commodities, mixed units, unnamed and explicit zero entries', async () => {
const input = tuple();
input[4] = [
{
@@ -96,7 +96,7 @@ it('preserves ordered duplicate commodities, mixed units, unnamed and explicit z
input[5] = [
{ acommodity: '', aquantity: { decimalMantissa: 0n, decimalPlaces: 2 }, acost: null }
];
const result = organize(input, 'assets:cash');
const result = await organize(input, 'assets:cash');
expect(result.change).toEqual([
{ commodity: 'USD', quantity: '9007199254740993.01', cost: null, costBasis: null, style: null },
{ commodity: 'USD', quantity: '-0.0001', cost: null, costBasis: null, style: null },
@@ -111,19 +111,23 @@ it.each(['{}', '[]', '[{}, {}, false, [], []]'])('rejects an incomplete tuple: %
expect(() => parseAccountRegisterRow(json)).toThrow('Prepared transactions could not be read.');
});
it('rejects unexpected contextual header differences rather than losing information', () => {
it('rejects unexpected contextual header differences rather than losing information', async () => {
const input = tuple();
input[1].tdate2 = 'unexpected';
expect(() => organize(input, 'assets:cash')).toThrow('Prepared transactions could not be read.');
await expect(organize(input, 'assets:cash')).rejects.toThrow(
'Prepared transactions could not be read.'
);
});
it('rejects contextual posting changes', () => {
it('rejects contextual posting changes', async () => {
const input = tuple();
input[1].tpostings.pop();
expect(() => organize(input, 'assets:cash')).toThrow('Prepared transactions could not be read.');
await expect(organize(input, 'assets:cash')).rejects.toThrow(
'Prepared transactions could not be read.'
);
});
it('recomputes selected indexes from original print, not stripped register posting positions', () => {
it('recomputes selected indexes from original print, not stripped register posting positions', async () => {
const input = tuple();
const print = structuredClone(input[0]);
print.tpostings.unshift({ ...print.tpostings[1], paccount: 'equity:conversion' });
@@ -132,7 +136,7 @@ it('recomputes selected indexes from original print, not stripped register posti
input,
'assets:cash',
transaction,
accountInvestmentContext([transaction], metadata, 'assets:cash')
await accountInvestmentContext([transaction], metadata, 'assets:cash')
);
expect(result.accountPostingIndexes).toEqual([1]);
expect(result.transaction).toBe(transaction);
@@ -141,20 +145,14 @@ it('recomputes selected indexes from original print, not stripped register posti
it.each(['tindex', 'ttags'] as const)(
'rejects a joined print whose stable %s identity differs',
(field) => {
async (field) => {
const input = tuple();
const print = structuredClone(input[0]);
if (field === 'tindex') print[field]++;
else print[field].push(['different', '']);
const transaction = prepareTransaction(print, structuredClone(print));
expect(() =>
organizeAccountRow(
input,
'assets:cash',
transaction,
accountInvestmentContext([transaction], metadata, 'assets:cash')
)
).toThrow('header');
const context = await accountInvestmentContext([transaction], metadata, 'assets:cash');
expect(() => organizeAccountRow(input, 'assets:cash', transaction, context)).toThrow('header');
}
);
@@ -163,7 +161,7 @@ it.each([
['counterpart', 'memo:outside', false]
] as const)(
'uses account-local historical virtual evidence (%s), not arbitrary counterpart virtual postings',
(_kind, virtualAccount, unclassified) => {
async (_kind, virtualAccount, unclassified) => {
const input = tuple();
input[5] = structuredClone(input[4]);
const transaction = prepareTransaction(input[0], structuredClone(input[0]));
@@ -172,7 +170,7 @@ it.each([
earlier.tpostings[1].paccount = virtualAccount;
earlier.tpostings[1].preal = 'VirtualPosting';
const history = prepareTransaction(earlier, structuredClone(earlier));
const context = accountInvestmentContext([history, transaction], metadata, 'assets:cash');
const context = await accountInvestmentContext([history, transaction], metadata, 'assets:cash');
const row = organizeAccountRow(input, 'assets:cash', transaction, context);
const amounts = [{ commodity: 'USD', quantity: '-100.00' }];
expect(row.investment.cash.change).toEqual(unclassified ? [] : amounts);
@@ -1,98 +1,26 @@
import { expect, it } from 'vitest';
import { AccountTransactionExports } from './export.js';
import { decodeNativeTransaction } from './export.js';
import { transactionFixture } from './transaction.fixture.js';
function native(index = 7, account = 'assets:cash') {
const transaction = transactionFixture();
transaction.tindex = index;
transaction.tpostings[0].paccount = account;
return transaction;
}
const json = (value: unknown) => JSON.stringify(value);
it('joins reordered exports by positive native index, not equal dates/descriptions/amounts', () => {
const exports = new AccountTransactionExports('assets:cash', false);
for (const index of [7, 9, 8]) exports.original(json(native(index)));
for (const index of [8, 7, 9]) exports.transacted(json(native(index)));
expect([...exports.transactions()].map((t) => t.index)).toEqual([7, 9, 8]);
expect([9, 8, 7].map((index) => exports.take(index).index)).toEqual([9, 8, 7]);
expect(() => exports.take(7)).toThrow('Missing or duplicate');
});
it('narrows CI print selection by exact canonical/subtree membership while retaining all counterparts', () => {
const exports = new AccountTransactionExports('assets:Cash', false);
const selected = native(1, 'assets:Cash:東京');
selected.tpostings.push({ ...selected.tpostings[1], paccount: 'assets:cash:other' });
const extras = [native(2, 'assets:cash'), native(3, 'assets:Cashbox')];
for (const transaction of [selected, ...extras]) exports.original(json(transaction));
for (const transaction of [...extras, selected]) exports.transacted(json(transaction));
expect([...exports.transactions()]).toHaveLength(1);
expect(exports.take(1).postings.map((p) => p.account)).toEqual([
'assets:Cash:東京',
'expenses:office',
'assets:cash:other'
]);
expect(() => exports.take(2)).toThrow();
});
it('drains ambiguous exports without projecting or storing them', () => {
const exports = new AccountTransactionExports('assets:cash', true);
const wire = json({ tindex: 7, tpostings: [{}] });
exports.original(wire);
exports.transacted(wire);
expect([...exports.transactions()]).toEqual([]);
expect(() => exports.original('{')).toThrow();
});
it.each([0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1])(
'rejects invalid index %s before storing',
(index) => {
const exports = new AccountTransactionExports('assets:cash', false);
expect(() => exports.original(json(native(index)))).toThrow();
expect(() => exports.transacted(json(native(index)))).toThrow();
}
);
it('rejects duplicate originals, duplicate transacted records, and missing joins', () => {
const exports = new AccountTransactionExports('assets:cash', false);
exports.original(json(native()));
expect(() => exports.original(json(native()))).toThrow('Duplicate');
expect(() => [...exports.transactions()]).toThrow('Missing native transacted');
expect(() => exports.transacted(json(native(8)))).toThrow('Missing native original');
exports.transacted(json(native()));
expect(() => exports.transacted(json(native()))).toThrow('Duplicate');
expect(() => exports.take(8)).toThrow();
});
it('does not attach mismatched header IDs or same-index changed metadata', () => {
const exports = new AccountTransactionExports('assets:cash', false);
exports.original(json(native()));
const changed = native();
changed.tcomment += 'changed';
expect(() => exports.transacted(json(changed))).toThrow('Mismatched');
});
it('joins exports containing native scientific integer mantissas without floating-point loss', () => {
const exports = new AccountTransactionExports('assets:cash', false);
const wire = json(native()).replace(
it('decodes native numerical tokens exactly without projecting away unknown fields', () => {
const native = { ...transactionFixture(), futureNativeField: { retained: true } };
const json = JSON.stringify(native).replace(
'"decimalMantissa":-10000',
'"decimalMantissa":-9.00719925474099301e17'
'"decimalMantissa":-900719925474099301e2'
);
exports.original(wire);
exports.transacted(wire);
const posting = exports.take(7).postings[0];
expect(posting.amounts[0].quantity).toBe('-9007199254740993.01');
expect(posting.transactedAmounts).toEqual([
{ commodity: 'USD', quantity: '-9007199254740993.01' }
]);
const decoded = decodeNativeTransaction(json);
expect(decoded.tpostings[0].pamount[0].aquantity.decimalMantissa).toBe(-90071992547409930100n);
expect(decoded).toMatchObject({
tindex: native.tindex,
tdescription: native.tdescription,
ttags: native.ttags,
tsourcepos: native.tsourcepos,
futureNativeField: { retained: true }
});
});
it.each(['original', 'transacted'] as const)(
'rejects fractional scientific mantissas in %s exports',
(kind) => {
const exports = new AccountTransactionExports('assets:cash', false);
exports.original(json(native()));
const malformed = json(native()).replace('"decimalMantissa":-10000', '"decimalMantissa":1e-20');
expect(() => exports[kind](malformed)).toThrow('integer mantissa');
}
);
it('rejects output that is not a native transaction record', () => {
expect(() => decodeNativeTransaction('[]')).toThrow();
expect(() => decodeNativeTransaction('{"tindex":1}')).toThrow();
expect(() => decodeNativeTransaction('{"tindex":0,"tpostings":[]}')).toThrow();
});
+1 -68
View File
@@ -1,14 +1,5 @@
import { parseExactJson } from '../amounts.js';
import {
prepareTransaction,
requireTransactionIndex,
type NativeTransaction,
type PreparedTransaction
} from './transaction.js';
type Entry =
| { kind: 'original'; transaction: NativeTransaction }
| { kind: 'joined'; transaction: PreparedTransaction };
import { requireTransactionIndex, type NativeTransaction } from './transaction.js';
export function decodeNativeTransaction(json: string): NativeTransaction {
const value = parseExactJson(json) as NativeTransaction;
@@ -22,61 +13,3 @@ export function decodeNativeTransaction(json: string): NativeTransaction {
requireTransactionIndex(value.tindex);
return value;
}
/** One account only. Never retain the wider case-insensitive print selection. */
export class AccountTransactionExports {
private readonly entries = new Map<number, Entry>();
private readonly account: string;
private readonly ambiguous: boolean;
constructor(account: string, ambiguous: boolean) {
this.account = account;
this.ambiguous = ambiguous;
}
private selected(transaction: NativeTransaction): boolean {
return transaction.tpostings.some(({ paccount }) => {
if (typeof paccount !== 'string') throw new Error('Invalid native posting account.');
return paccount === this.account || paccount.startsWith(`${this.account}:`);
});
}
original(json: string): void {
const transaction = decodeNativeTransaction(json);
if (this.ambiguous || !this.selected(transaction)) return;
if (this.entries.has(transaction.tindex)) throw new Error('Duplicate native original index.');
// Keep the native original until its non-amount structure can be checked.
this.entries.set(transaction.tindex, { kind: 'original', transaction });
}
transacted(json: string): void {
const native = decodeNativeTransaction(json);
if (this.ambiguous) return;
const entry = this.entries.get(native.tindex);
if (!entry) {
if (this.selected(native)) throw new Error('Missing native original index.');
return;
}
if (entry.kind !== 'original') throw new Error('Duplicate native transacted index.');
const transaction = prepareTransaction(entry.transaction, native);
// Replace the raw original only after validating the joined exports.
this.entries.set(native.tindex, { kind: 'joined', transaction });
}
*transactions(): IterableIterator<PreparedTransaction> {
for (const entry of this.entries.values()) {
if (entry.kind !== 'joined') throw new Error('Missing native transacted index.');
yield entry.transaction;
}
}
take(index: number): PreparedTransaction {
requireTransactionIndex(index);
const entry = this.entries.get(index);
if (!entry || entry.kind !== 'joined') {
throw new Error('Missing or duplicate native register index.');
}
this.entries.delete(index);
return entry.transaction;
}
}
@@ -1,3 +1,4 @@
export { decodeNativeTransaction } from './export.js';
export { assertTransactionExportsMatch } from './transaction.js';
export type { NativeTransaction } from './transaction.js';
export type { NativeTransactionPair, PreparationTransactions } from './store.js';
@@ -112,7 +112,7 @@ function cashEvidence(commodity = 'USD'): PreparedTransaction {
]);
}
function prepare(
async function prepare(
input: Omit<PreparedAccountRow, 'investment'>,
related: PreparedTransaction[] = [],
roles = metadata(),
@@ -120,11 +120,11 @@ function prepare(
) {
return prepareInvestmentActivity(
input,
accountInvestmentContext([...related, input.transaction], roles, canonicalAccount)
await accountInvestmentContext([...related, input.transaction], roles, canonicalAccount)
);
}
it('keeps externally funded two-security purchases together without duplicating account cash', () => {
it('keeps externally funded two-security purchases together without duplicating account cash', async () => {
const input = row(
[
posting('holdings', [security('VTI', '10.00')], 'acquire', [value('USD', '1000')]),
@@ -137,7 +137,7 @@ it('keeps externally funded two-security purchases together without duplicating
[amount('VTI', '10.00'), amount('BND', '5.00')],
[amount('VTI', '10.00'), amount('BND', '5.00')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities).toEqual([
{
commodity: 'VTI',
@@ -167,7 +167,7 @@ it('keeps externally funded two-security purchases together without duplicating
expect(result.issues).toEqual([]);
});
it('keeps internal cash change and the final native balance once, not per security', () => {
it('keeps internal cash change and the final native balance once, not per security', async () => {
const input = row(
[
posting('holdings', [security('VTI', '10.00')], 'acquire', [value('USD', '1000')]),
@@ -178,7 +178,7 @@ it('keeps internal cash change and the final native balance once, not per securi
[amount('VTI', '10.00'), amount('USD', '-1400.00'), amount('BND', '5.00')],
[amount('VTI', '20.00'), amount('USD', '600.00'), amount('BND', '5.00')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities.map(({ action }) => action)).toEqual(['buy', 'buy']);
expect(result.cash).toEqual({
direction: 'out',
@@ -193,7 +193,7 @@ it('keeps internal cash change and the final native balance once, not per securi
);
});
it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separate', () => {
it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separate', async () => {
const gain = posting('gains', [amount('USD', '-250')], 'gain');
gain.tags.push(['_generated-posting', '']);
const input = row(
@@ -208,7 +208,7 @@ it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separa
[amount('VTI', '-5'), amount('USD', '750.00')],
[amount('VTI', '15'), amount('USD', '950.00')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities[0]).toMatchObject({
action: 'sell',
prices: [value('USD', '150.00')],
@@ -223,7 +223,7 @@ it('uses native sale proceeds, not basis, and keeps overlapping R/G gains separa
it.each(['UnitCost', 'TotalCost'] as const)(
'retains native %s values without deriving a unit price',
(kind) => {
async (kind) => {
const shares = security('VTI', '0.000000000000000001');
shares.cost = { kind, amount: amount('USD', '9007199254740993.0001') };
const input = row(
@@ -234,7 +234,7 @@ it.each(['UnitCost', 'TotalCost'] as const)(
[0],
[amount('VTI', '0.000000000000000001')]
);
const result = prepare(input).securities[0];
const result = (await prepare(input)).securities[0];
expect(result.transactedAmounts).toEqual([value('USD', '12345678901234567890.00000001')]);
expect(result.prices).toEqual(
kind === 'UnitCost' ? [value('USD', '9007199254740993.0001')] : []
@@ -262,7 +262,7 @@ it.each([
}
])(
'keeps native TotalCost $role value distinct from net settlement and expense',
({ role, units, transacted, cash, expense, action }) => {
async ({ role, units, transacted, cash, expense, action }) => {
// Values observed in the local native edge probe, not calculated by this preparation layer.
const shares = security('XYZ', units, null, '40');
shares.cost = { kind: 'TotalCost', amount: amount('USD', transacted) };
@@ -277,7 +277,7 @@ it.each([
postings.push(gain);
}
const input = row(postings, [0, 1], [amount('XYZ', units), amount('USD', cash)]);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities).toEqual([
{
commodity: 'XYZ',
@@ -297,7 +297,7 @@ it.each([
}
);
it('keeps the native income/acquisition co-occurrence without manufacturing cash movement', () => {
it('keeps the native income/acquisition co-occurrence without manufacturing cash movement', async () => {
const input = row(
[
posting('holdings', [security('XYZ', '1', '50.00', '50.00')], 'acquire', [
@@ -309,7 +309,7 @@ it('keeps the native income/acquisition co-occurrence without manufacturing cash
[amount('XYZ', '1')],
[amount('USD', '895.00'), amount('XYZ', '4')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities[0]).toMatchObject({
action: 'acquire',
prices: [value('USD', '50.00')],
@@ -327,7 +327,7 @@ it('keeps the native income/acquisition co-occurrence without manufacturing cash
expect(result.issues).toEqual(['unattributed-income']);
});
it('concatenates complete native value bags without zipping, summing or normalizing quantities', () => {
it('concatenates complete native value bags without zipping, summing or normalizing quantities', async () => {
const input = row(
[
posting('holdings', [security('BND', '1')], 'acquire', [value('USD', '10.0')]),
@@ -344,7 +344,7 @@ it('concatenates complete native value bags without zipping, summing or normaliz
[3, 2, 1, 0],
[amount('VTI', '6.000'), amount('BND', '3'), amount('VTI', '0.00')]
);
const result = prepare(input);
const result = await prepare(input);
expect(
result.securities.map(({ commodity, postingIndexes }) => ({ commodity, postingIndexes }))
).toEqual([
@@ -363,23 +363,26 @@ it('concatenates complete native value bags without zipping, summing or normaliz
it.each([
['transfer-from', '-2.00', 'transfer-out'],
['transfer-to', '2.00', 'transfer-in']
] as const)('keeps %s units distinct from transacted monetary values', (role, quantity, action) => {
const input = row(
[posting('holdings', [security('VTI', quantity, null)], role, [value('VTI', quantity)])],
[0],
[amount('VTI', quantity)]
);
expect(prepare(input).securities[0]).toEqual({
commodity: 'VTI',
action,
change: [value('VTI', quantity)],
postingIndexes: [0],
prices: [],
transactedAmounts: null
});
});
] as const)(
'keeps %s units distinct from transacted monetary values',
async (role, quantity, action) => {
const input = row(
[posting('holdings', [security('VTI', quantity, null)], role, [value('VTI', quantity)])],
[0],
[amount('VTI', quantity)]
);
expect((await prepare(input)).securities[0]).toEqual({
commodity: 'VTI',
action,
change: [value('VTI', quantity)],
postingIndexes: [0],
prices: [],
transactedAmounts: null
});
}
);
it('does not turn a native omitted converted posting into a zero or a computed value', () => {
it('does not turn a native omitted converted posting into a zero or a computed value', async () => {
const input = row(
[
posting('holdings', [security('VTI', '10')], 'acquire', null),
@@ -388,7 +391,7 @@ it('does not turn a native omitted converted posting into a zero or a computed v
[0],
[amount('VTI', '10')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0]).toMatchObject({ action: 'acquire', transactedAmounts: null });
expect(result.issues).toContain('unavailable-transacted-values');
expect(input.transaction.postings[0].transactedAmounts).toBeNull();
@@ -396,7 +399,7 @@ it('does not turn a native omitted converted posting into a zero or a computed v
it.each<{ missing: ReportedAmount[] | null }>([{ missing: null }, { missing: [] }])(
'does not attribute a partial value bag when any contributing posting is unavailable: %j',
({ missing }) => {
async ({ missing }) => {
const input = row(
[
posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]),
@@ -406,11 +409,11 @@ it.each<{ missing: ReportedAmount[] | null }>([{ missing: null }, { missing: []
[0, 1],
[amount('VTI', '3')]
);
expect(prepare(input).securities[0].transactedAmounts).toBeNull();
expect((await prepare(input)).securities[0].transactedAmounts).toBeNull();
}
);
it('does not apportion a converted mixed-security posting by original amount indexes', () => {
it('does not apportion a converted mixed-security posting by original amount indexes', async () => {
const input = row(
[
posting('holdings', [security('VTI', '10'), security('BND', '5', '80.00')], 'acquire', [
@@ -421,7 +424,7 @@ it('does not apportion a converted mixed-security posting by original amount ind
[0],
[amount('VTI', '10'), amount('BND', '5')]
);
const result = prepare(input);
const result = await prepare(input);
expect(
result.securities.map(({ action, transactedAmounts }) => ({ action, transactedAmounts }))
).toEqual([
@@ -431,7 +434,7 @@ it('does not apportion a converted mixed-security posting by original amount ind
expect(result.issues).toContain('mixed-security-posting');
});
it('also refuses attribution when a single posting mixes security and cash amounts', () => {
it('also refuses attribution when a single posting mixes security and cash amounts', async () => {
const input = row(
[
posting('holdings', [security('VTI', '1'), amount('USD', '-100')], 'acquire', [
@@ -441,12 +444,12 @@ it('also refuses attribution when a single posting mixes security and cash amoun
[0],
[amount('VTI', '1'), amount('USD', '-100')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities[0].transactedAmounts).toBeNull();
expect(result.issues).toContain('mixed-security-posting');
});
it('retains all expense evidence without inventing commissions or security allocation', () => {
it('retains all expense evidence without inventing commissions or security allocation', async () => {
const input = row(
[
posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]),
@@ -457,14 +460,14 @@ it('retains all expense evidence without inventing commissions or security alloc
[0],
[amount('VTI', '1')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.expensePostingIndexes).toEqual([2, 3]);
expect(result.cashPostingIndexes).toEqual([1]);
expect(result.securities[0].transactedAmounts).toEqual([value('USD', '100')]);
expect(result.securities[0]).not.toHaveProperty('commission');
});
it('leaves dividend-like cash receipts generic and unattributed', () => {
it('leaves dividend-like cash receipts generic and unattributed', async () => {
const input = row(
[posting('holdings', [amount('USD', '25.00')]), posting('revenue', [amount('USD', '-25.00')])],
[0],
@@ -472,7 +475,7 @@ it('leaves dividend-like cash receipts generic and unattributed', () => {
[amount('USD', '225.00')]
);
input.transaction.description = 'VTI Dividend';
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.securities).toEqual([]);
expect(result.incomePostingIndexes).toEqual([1]);
expect(result.issues).toEqual(['unattributed-income']);
@@ -484,7 +487,7 @@ it.each([
{ role: 'dispose', units: '-1', transacted: '-100', counterpart: '100', cash: '5', expense: '-5' }
])(
'does not promote $role with equity/unknown funding merely because a cash fee leg exists',
({ role, units, transacted, counterpart, cash, expense }) => {
async ({ role, units, transacted, counterpart, cash, expense }) => {
const opening = posting('opening-or-gift', [amount('USD', counterpart)]);
opening.tags.push(['type', 'E']);
const input = row(
@@ -497,7 +500,7 @@ it.each([
[0],
[amount('VTI', units)]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0]).toMatchObject({
action: role,
transactedAmounts: [value('USD', transacted)]
@@ -507,11 +510,11 @@ it.each([
// An untyped counterpart is equally insufficient; descriptions and names cannot help.
opening.tags = [];
input.transaction.description = 'Buy VTI';
expect(prepare(input).securities[0].action).toBe(role);
expect((await prepare(input)).securities[0].action).toBe(role);
}
);
it('retains income plus acquisition without inventing reinvestment or cash funding allocation', () => {
it('retains income plus acquisition without inventing reinvestment or cash funding allocation', async () => {
const input = row(
[
posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]),
@@ -521,14 +524,14 @@ it('retains income plus acquisition without inventing reinvestment or cash fundi
[0],
[amount('VTI', '1')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0].action).toBe('acquire');
expect(result.incomePostingIndexes).toEqual([1]);
expect(result.cashPostingIndexes).toEqual([2]);
expect(result.issues).toContain('unattributed-income');
});
it('recognizes gain roles outside the native G set and excludes generated income', () => {
it('recognizes gain roles outside the native G set and excludes generated income', async () => {
const generated = posting('revenue', [amount('USD', '-2')]);
generated.tags.push(['_generated-posting', '']);
const input = row(
@@ -540,12 +543,12 @@ it('recognizes gain roles outside the native G set and excludes generated income
[0],
[amount('USD', '5')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.gainPostingIndexes).toEqual([1]);
expect(result.incomePostingIndexes).toEqual([]);
});
it('requires account-local real Cash-type evidence, not currency spelling or price currency', () => {
it('requires account-local real Cash-type evidence, not currency spelling or price currency', async () => {
const input = row(
[
posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]),
@@ -555,7 +558,7 @@ it('requires account-local real Cash-type evidence, not currency spelling or pri
[amount('VTI', '1'), amount('USD', '-100')],
[amount('USD', '200')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0].action).toBe('acquire');
expect(result.cash).toEqual({
direction: 'unknown',
@@ -567,7 +570,7 @@ it('requires account-local real Cash-type evidence, not currency spelling or pri
expect(result.cashPostingIndexes).toEqual([]);
});
it('does not infer non-lot securities from ticker, account names, prices or untrusted type tags', () => {
it('does not infer non-lot securities from ticker, account names, prices or untrusted type tags', async () => {
const priced = amount('AAPL', '2');
priced.cost = { kind: 'UnitCost', amount: amount('USD', '100') };
const stock = posting('assets:investments:stocks', [priced], 'acquire', [value('USD', '200')]);
@@ -577,13 +580,13 @@ it('does not infer non-lot securities from ticker, account names, prices or untr
[0],
[amount('AAPL', '2')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities).toEqual([]);
expect(result.cash.unclassifiedChange).toEqual([value('AAPL', '2')]);
expect(result.issues).toContain('unclassified-commodity');
});
it('allows explicit processed basis plus a recognized role to establish an undeclared symbol', () => {
it('allows explicit processed basis plus a recognized role to establish an undeclared symbol', async () => {
const input = row(
[
posting('holdings', [security('not a ticker', '2')], 'acquire', [value('USD', '200')]),
@@ -592,10 +595,13 @@ it('allows explicit processed basis plus a recognized role to establish an undec
[0],
[amount('not a ticker', '2')]
);
expect(prepare(input).securities[0]).toMatchObject({ commodity: 'not a ticker', action: 'buy' });
expect((await prepare(input)).securities[0]).toMatchObject({
commodity: 'not a ticker',
action: 'buy'
});
});
it('treats evidenced FX as mixed cash, not a security just because it has a cost', () => {
it('treats evidenced FX as mixed cash, not a security just because it has a cost', async () => {
const euros = amount('EUR', '90.00');
euros.cost = { kind: 'TotalCost', amount: amount('USD', '100.00') };
const input = row(
@@ -604,7 +610,7 @@ it('treats evidenced FX as mixed cash, not a security just because it has a cost
[amount('EUR', '90.00'), amount('USD', '-100.00')],
[amount('EUR', '90.00')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities).toEqual([]);
expect(result.cash.direction).toBe('mixed');
expect(result.cash.change).toEqual([value('EUR', '90.00'), value('USD', '-100.00')]);
@@ -618,7 +624,7 @@ it.each([
['holdings', 'dispose', '-2', [amount('VTI', '-1')]]
] as const)(
'does not label unsupported short evidence as an ordinary trade (%s/%s/%s)',
(account, role, quantity, balance) => {
async (account, role, quantity, balance) => {
const input = row(
[
posting(account, [security('VTI', quantity)], role, [value('USD', '-200')]),
@@ -628,13 +634,13 @@ it.each([
[amount('VTI', quantity)],
[...balance]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0].action).toBe('unknown');
expect(result.issues).toContain('unsupported-short');
}
);
it('does not treat liability cash as an asset even if native account-query sets overlap', () => {
it('does not treat liability cash as an asset even if native account-query sets overlap', async () => {
const roles = metadata();
roles.assets = new Set([...roles.assets, 'shorts']);
roles.cash = new Set([...roles.cash, 'shorts']);
@@ -644,7 +650,7 @@ it('does not treat liability cash as an asset even if native account-query sets
[amount('USD', '-10')],
[amount('USD', '-30')]
);
const result = prepare(input, [], roles);
const result = await prepare(input, [], roles);
expect(result.cash.change).toEqual([]);
expect(result.cash.unclassifiedChange).toEqual([value('USD', '-10')]);
expect(result.cashPostingIndexes).toEqual([1]);
@@ -652,7 +658,7 @@ it('does not treat liability cash as an asset even if native account-query sets
it.each(['VirtualPosting', 'BalancedVirtualPosting'])(
'does not promote %s security or mixed cash to ordinary activity',
(kind) => {
async (kind) => {
const virtual = posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]);
virtual.kind = kind;
const input = row(
@@ -665,7 +671,7 @@ it.each(['VirtualPosting', 'BalancedVirtualPosting'])(
[amount('VTI', '1'), amount('USD', '-100')],
[amount('USD', '20')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0].action).toBe('unknown');
expect(result.cash.change).toEqual([]);
expect(result.cash.unclassifiedChange).toEqual([value('USD', '-100')]);
@@ -682,7 +688,7 @@ it.each([
{ account: 'elsewhere', kind: 'BalancedVirtualPosting', contaminated: false }
])(
'uses scoped virtual history for later cash bags: $account/$kind',
({ account, kind, contaminated }) => {
async ({ account, kind, contaminated }) => {
const virtual = posting(account, [amount('USD', '10.00')]);
virtual.kind = kind;
const prior = transaction([posting('holdings', [amount('USD', '0.00')]), virtual]);
@@ -693,7 +699,7 @@ it.each([
[amount('USD', '5.00')],
[amount('USD', '15.00')]
);
const context = accountInvestmentContext(
const context = await accountInvestmentContext(
[cashEvidence(), prior, input.transaction],
metadata(),
'holdings'
@@ -727,7 +733,7 @@ it.each([
it.each(['liabilities', 'income', 'expenses', 'gains', 'unknown'] as const)(
'keeps later parent cash bags unclassified after selected %s history',
(role) => {
async (role) => {
const roles = metadata();
roles.assets = new Set([...roles.assets, 'portfolio:cash']);
roles.cash = new Set([...roles.cash, 'portfolio:cash']);
@@ -747,7 +753,7 @@ it.each(['liabilities', 'income', 'expenses', 'gains', 'unknown'] as const)(
[amount('USD', '100.00')]
);
const before = structuredClone({ prior, input, roles });
const result = prepare(input, [prior], roles, 'portfolio');
const result = await prepare(input, [prior], roles, 'portfolio');
expect(result.cash).toEqual({
direction: 'unknown',
change: [],
@@ -769,7 +775,7 @@ it.each([
{ account: 'elsewhere', contaminated: false }
])(
'scopes non-asset history to the canonical account/subtree: $account',
({ account, contaminated }) => {
async ({ account, contaminated }) => {
const roles = metadata();
roles.assets = new Set([...roles.assets, 'portfolio:cash']);
roles.cash = new Set([...roles.cash, 'portfolio:cash']);
@@ -781,7 +787,7 @@ it.each([
[amount('USD', '200.00')],
[amount('USD', contaminated ? '100.00' : '200.00')]
);
const context = accountInvestmentContext([prior, input.transaction], roles, 'portfolio');
const context = await accountInvestmentContext([prior, input.transaction], roles, 'portfolio');
expect(context.hasNonAssetAccountPostings).toBe(contaminated);
const result = prepareInvestmentActivity(input, context);
expect(result.cash.direction).toBe(contaminated ? 'unknown' : 'in');
@@ -791,11 +797,11 @@ it.each([
}
);
it('does not use virtual Cash postings as commodity evidence', () => {
it('does not use virtual Cash postings as commodity evidence', async () => {
const virtual = posting('wallet', [amount('unusual cash unit', '10')]);
virtual.kind = 'VirtualPosting';
const input = row([virtual], [0], [amount('unusual cash unit', '10')]);
const result = prepare(input);
const result = await prepare(input);
expect(result.cashPostingIndexes).toEqual([]);
expect(result.cash.unclassifiedChange).toEqual([value('unusual cash unit', '10')]);
});
@@ -815,26 +821,26 @@ it.each<{ tags: [string, string][] }>([
['_ptype', 'gain']
]
}
])('defers missing, unknown and conflicting security roles: %j', ({ tags }) => {
])('defers missing, unknown and conflicting security roles: %j', async ({ tags }) => {
const stock = posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100')]);
stock.tags = tags;
const input = row([stock, posting('wallet', [amount('USD', '-100')])], [0], [amount('VTI', '1')]);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0].action).toBe('unknown');
expect(result.issues).toContain('unknown-native-role');
});
it('does not silently let conflicting role tags establish a security from basis alone', () => {
it('does not silently let conflicting role tags establish a security from basis alone', async () => {
const stock = posting('holdings', [security('undeclared', '1')], 'acquire');
stock.tags.push(['_ptype', 'dispose']);
const input = row([stock], [0], [amount('undeclared', '1')]);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities).toEqual([]);
expect(result.issues).toContain('unknown-native-role');
expect(result.cash.unclassifiedChange).toEqual([value('undeclared', '1')]);
});
it('marks opposing native actions mixed and uses only the whole-transaction net change', () => {
it('marks opposing native actions mixed and uses only the whole-transaction net change', async () => {
const input = row(
[
posting('holdings', [security('VTI', '-2', '110')], 'dispose', [value('USD', '-220')]),
@@ -845,7 +851,7 @@ it('marks opposing native actions mixed and uses only the whole-transaction net
[amount('VTI', '1.00')],
[amount('VTI', '8.00')]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0]).toMatchObject({
action: 'mixed',
change: [value('VTI', '1.00')],
@@ -855,7 +861,7 @@ it('marks opposing native actions mixed and uses only the whole-transaction net
expect(result.issues).toContain('mixed-security-actions');
});
it('retains both sides of a subtree share transfer without fabricating a net zero', () => {
it('retains both sides of a subtree share transfer without fabricating a net zero', async () => {
const input = row(
[
posting('holdings', [security('VTI', '-2', null)], 'transfer-from'),
@@ -864,7 +870,7 @@ it('retains both sides of a subtree share transfer without fabricating a net zer
[0, 1],
[]
);
const result = prepare(input);
const result = await prepare(input);
expect(result.securities[0]).toMatchObject({
action: 'mixed',
change: [],
@@ -873,7 +879,7 @@ it('retains both sides of a subtree share transfer without fabricating a net zer
expect(result.cash.change).toEqual([]);
});
it('keeps costless acquisitions and disposals neutral despite counterpart cash', () => {
it('keeps costless acquisitions and disposals neutral despite counterpart cash', async () => {
const input = row(
[
posting('holdings', [security('VTI', '-1', null)], 'dispose'),
@@ -882,7 +888,7 @@ it('keeps costless acquisitions and disposals neutral despite counterpart cash',
[0],
[amount('VTI', '-1')]
);
expect(prepare(input).securities[0]).toMatchObject({
expect((await prepare(input)).securities[0]).toMatchObject({
action: 'dispose',
prices: [],
transactedAmounts: null
@@ -897,9 +903,9 @@ it.each([
[[amount('USD', '10.00'), amount('USD', '-1.00')], 'mixed']
] as const)(
'reads cash direction from exact signs without aggregating: %j',
(change, direction) => {
async (change, direction) => {
const input = row([posting('holdings', [amount('USD', '0')])], [0], [...change]);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.cash.direction).toBe(direction);
expect(result.cash.change).toEqual(
change.map(({ commodity, quantity }) => ({ commodity, quantity }))
@@ -908,9 +914,9 @@ it.each([
}
);
it('keeps unidentified zero units and balances visible, without inventing a currency zero', () => {
it('keeps unidentified zero units and balances visible, without inventing a currency zero', async () => {
const input = row([posting('holdings', [])], [0], [amount('', '0.00')], [amount('mystery', '2')]);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.cash).toEqual({
direction: 'unknown',
change: [],
@@ -920,19 +926,19 @@ it('keeps unidentified zero units and balances visible, without inventing a curr
});
});
it('does not let an unclassified balance alone change a known cash movement direction', () => {
it('does not let an unclassified balance alone change a known cash movement direction', async () => {
const input = row(
[posting('holdings', [amount('USD', '1')])],
[0],
[amount('USD', '1')],
[amount('USD', '10'), amount('mystery', '2'), amount('VTI', '5')]
);
const result = prepare(input, [cashEvidence()]);
const result = await prepare(input, [cashEvidence()]);
expect(result.cash.direction).toBe('in');
expect(result.cash.unclassifiedBalance).toEqual([value('mystery', '2')]);
});
it('builds context in one pass without mutating metadata and excludes later discovered securities from cash', () => {
it('builds context in one pass without mutating metadata and excludes later discovered securities from cash', async () => {
const roles = metadata();
const before = structuredClone(roles);
const early = transaction([
@@ -945,14 +951,14 @@ it('builds context in one pass without mutating metadata and excludes later disc
yield early;
yield later;
}
const context = accountInvestmentContext(related(), roles, 'holdings');
const context = await accountInvestmentContext(related(), roles, 'holdings');
expect(context.securityCommodities).toEqual(new Set(['VTI', 'BND', 'special']));
expect(context.cashCommodities).toEqual(new Set(['tokens']));
expect(roles).toEqual(before);
expect(roles.lotCommodities.has('special')).toBe(false);
});
it('preparation leaves the source row, tags, annotations, amount bags and context untouched', () => {
it('preparation leaves the source row, tags, annotations, amount bags and context untouched', async () => {
const input = row(
[
posting('holdings', [security('VTI', '1')], 'acquire', [value('USD', '100.0000')]),
@@ -962,7 +968,7 @@ it('preparation leaves the source row, tags, annotations, amount bags and contex
[amount('VTI', '1.0000')],
[amount('VTI', '4.0000')]
);
const context = accountInvestmentContext([input.transaction], metadata(), 'holdings');
const context = await accountInvestmentContext([input.transaction], metadata(), 'holdings');
const before = structuredClone({ input, context });
const result = prepareInvestmentActivity(input, context);
expect({ input, context }).toEqual(before);
@@ -108,16 +108,16 @@ function isAsset(posting: PreparedPosting, metadata: InvestmentMetadata): boolea
}
/** Only related original transactions establish cash units; annotations are not cash legs. */
export function accountInvestmentContext(
transactions: Iterable<PreparedTransaction>,
export async function accountInvestmentContext(
transactions: Iterable<PreparedTransaction> | AsyncIterable<PreparedTransaction>,
metadata: InvestmentMetadata,
canonicalAccount: string
): AccountInvestmentContext {
): Promise<AccountInvestmentContext> {
const securityCommodities = new Set(metadata.lotCommodities);
const cashCandidates = new Set<string>();
let hasVirtualAccountPostings = false;
let hasNonAssetAccountPostings = false;
for (const transaction of transactions) {
for await (const transaction of transactions) {
for (const posting of transaction.postings) {
if (
posting.account === canonicalAccount ||
@@ -0,0 +1,13 @@
export interface NativeTransactionPair {
originalJson: string;
transactedJson: string;
}
/** Caller-owned storage for one explicit preparation; never a journal writer. */
export interface PreparationTransactions {
insertOriginal(json: string): Promise<void>;
attachTransacted(json: string): Promise<void>;
assertComplete(): Promise<void>;
read(nativeIndex: number): Promise<NativeTransactionPair>;
accountHistory(accountName: string): AsyncIterable<NativeTransactionPair>;
}
@@ -1,5 +1,6 @@
import * as databases from '@hledger-reports/db';
import * as reports from '@hledger-reports/db/reports';
import { decodeNativeTransaction } from '@hledger-reports/hledger/transactions';
import * as crypto from 'node:crypto';
import * as fs from 'node:fs/promises';
import { tmpdir } from 'node:os';
@@ -20,6 +21,10 @@ import {
ready,
preparedAccountRows as rows
} from '../../../db/src/reports/data.fixture.js';
import {
nativeTransactionFixture,
richNativeExportsFixture
} from '../../../db/src/transactions/native.fixture.js';
vi.mock('node:fs/promises', async (original) => ({
...(await original<typeof import('node:fs/promises')>())
@@ -164,6 +169,92 @@ it('commits and closes a complete private database before making its directory v
expect((await fs.stat(join(publication.directory, 'reports.sqlite'))).mode & 0o777).toBe(0o600);
});
it('publishes the stage-bound native import without changing either raw export', async () => {
const pair = richNativeExportsFixture();
const stage = await createReportStage(snapshots, older);
const { insertOriginal, attachTransacted, assertComplete, read, accountHistory } =
stage.transactions;
await insertOriginal(pair.originalJson);
await attachTransacted(pair.transactedJson);
await assertComplete();
expect(await read(7)).toEqual(pair);
const history: Array<typeof pair> = [];
for await (const transaction of accountHistory('assets:cash')) history.push(transaction);
expect(history).toEqual([pair]);
await stage.publish(fixture(), [], journal);
const publication = await selected();
await expect(insertOriginal(pair.originalJson)).rejects.toSatisfy(
safeError('Unable to prepare reports')
);
await stage.discard();
const database = new DatabaseSync(join(publication.directory, 'reports.sqlite'), {
readOnly: true
});
try {
expect(
database
.prepare(
'SELECT native_index, original_json, transacted_json FROM transactions ORDER BY native_index'
)
.all()
).toEqual([
{
native_index: 7,
original_json: pair.originalJson,
transacted_json: pair.transactedJson
}
]);
} finally {
database.close();
}
});
it('pages complete native history and lets caller cancellation run between batches', async () => {
const stage = await createReportStage(snapshots, older);
let abort: ReturnType<typeof setImmediate> | undefined;
try {
const expected = Array.from({ length: 52 }, (_, index) => index + 1);
for (const index of expected) {
const json = JSON.stringify(nativeTransactionFixture(index));
await stage.transactions.insertOriginal(json);
await stage.transactions.attachTransacted(json);
}
await stage.transactions.assertComplete();
const controller = new AbortController();
const interrupted: number[] = [];
for await (const pair of stage.transactions.accountHistory('assets:cash')) {
if (controller.signal.aborted) break;
interrupted.push(decodeNativeTransaction(pair.originalJson).tindex);
if (interrupted.length === 1) abort = setImmediate(() => controller.abort());
}
expect(interrupted.length).toBeGreaterThan(0);
expect(interrupted.length).toBeLessThan(expected.length);
const complete: number[] = [];
for await (const pair of stage.transactions.accountHistory('assets:cash'))
complete.push(decodeNativeTransaction(pair.originalJson).tindex);
expect(complete).toEqual(expected);
} finally {
if (abort) clearImmediate(abort);
await stage.discard();
}
});
it('rejects an incomplete native import without replacing the last good publication', async () => {
await publish();
const previous = await selected();
const stage = await createReportStage(snapshots, newer);
try {
await stage.transactions.insertOriginal(richNativeExportsFixture().originalJson);
await expect(stage.publish(fixture(), [], journal)).rejects.toSatisfy(
safeError('Unable to prepare reports')
);
expect(await selected()).toEqual(previous);
expect(await fs.readdir(join(snapshots, '.sets'))).toEqual([basename(previous.directory)]);
} finally {
await stage.discard();
}
});
it('distinguishes empty-ready accounts from ambiguous access without requiring empty callbacks', async () => {
const stage = await createReportStage(snapshots, older);
await stage.writeRows('empty callback', []);
+49 -1
View File
@@ -1,8 +1,10 @@
import type {
OverviewData,
PreparedAccountSummary,
PreparedAccountRow
PreparedAccountRow,
PreparationTransactions
} from '@hledger-reports/hledger';
import type { NativeTransactionPair } from '@hledger-reports/hledger/transactions';
import {
createReportDatabase,
openReadOnlyReportDatabase,
@@ -29,9 +31,18 @@ import {
type StoredAccount,
type StoredReportData
} from '@hledger-reports/db/reports';
import {
insertNativeTransaction,
attachTransactedTransaction,
assertTransactionImportComplete,
readNativeTransaction,
readAccountTransactionBatch,
type StoredNativeTransaction
} from '@hledger-reports/db/transactions';
import { randomInt, randomUUID } from 'node:crypto';
import { lstat, mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
import { basename, dirname, join } from 'node:path';
import { setImmediate } from 'node:timers/promises';
import { namesIn, privateDirectory } from '../files.js';
const runIdPattern = /^[A-Za-z0-9_-]{1,128}(?![\s\S])/;
@@ -45,6 +56,7 @@ export interface ReportPublication extends StoredReportData {
}
export interface ReportStage {
transactions: PreparationTransactions;
writeRows(name: string, rows: readonly PreparedAccountRow[]): Promise<void>;
publish(
overview: OverviewData,
@@ -54,6 +66,12 @@ export interface ReportStage {
discard(): Promise<void>;
}
function nativePair(transaction: StoredNativeTransaction | null): NativeTransactionPair {
if (!transaction || transaction.transactedJson === null)
throw new Error('Native transaction exports are unavailable.');
return { originalJson: transaction.originalJson, transactedJson: transaction.transactedJson };
}
function requestCreationKey(token: string): string {
// The separator in latestSet preserves request ordering for prefix run IDs.
return setPattern.exec(token)![1];
@@ -234,6 +252,35 @@ export async function createReportStage(
}
}
return {
transactions: {
insertOriginal(json) {
return operation(() => insertNativeTransaction(writer, json));
},
attachTransacted(json) {
return operation(() => attachTransactedTransaction(writer, json));
},
assertComplete() {
return operation(() => assertTransactionImportComplete(writer));
},
read(nativeIndex) {
return operation(async () =>
nativePair(await readNativeTransaction(writer, nativeIndex))
);
},
async *accountHistory(accountName) {
let afterNativeIndex = 0;
while (true) {
const batch = await operation(() =>
readAccountTransactionBatch(writer, accountName, afterNativeIndex)
);
if (batch.length === 0) return;
for (const transaction of batch) yield nativePair(transaction);
afterNativeIndex = batch[batch.length - 1].nativeIndex;
// SQLite runs synchronously; let caller abort callbacks run between batches.
await setImmediate();
}
}
},
writeRows(name, rows) {
return operation(async () => {
if (typeof name !== 'string' || !Array.isArray(rows)) throw new Error();
@@ -292,6 +339,7 @@ export async function createReportStage(
});
}
if (remaining.size) throw new Error();
await assertTransactionImportComplete(writer);
await writeReportData(writer, { overview, accounts: catalog, journal });
await writeAccountActivitySummaries(writer, activitySummaries);
await writer.commit().execute();