Read shared native transactions in saved registers #51

Merged
eleith merged 1 commits from feat/shared-account-transactions into main 2026-11-12 17:30:41 +00:00
13 changed files with 594 additions and 199 deletions

No files matched your search

@@ -0,0 +1,6 @@
---
'@hledger-reports/web': patch
'@hledger-reports/worker': patch
---
Read saved registers from shared native transactions instead of per-account copies and remove legacy financial JSON-file recognition. Prepare reports again after updating.
@@ -6,31 +6,54 @@ import type { PreparedAccountSummary } from '@hledger-reports/hledger';
import { createReportStage, readReportPublication } from '@hledger-reports/snapshots';
import {
amount,
preparedAccountRows,
preparedAccountRowsFromExports,
preparedOverview
} from '../../../../../packages/db/src/reports/data.fixture.js';
import { nativeTransactionFixture } from '../../../../../packages/db/src/transactions/native.fixture.js';
import { load as loadLayout } from '../../routes/(protected)/+layout.server.js';
import { load as loadCashAccount } from '../../routes/(protected)/registers/cash/[accountId]/+page.server.js';
import { load as loadInvestmentAccount } from '../../routes/(protected)/registers/investments/[accountId]/+page.server.js';
function activity(account: string, payee: string, visibleBalance: string, endingBalance: string) {
const rows = preparedAccountRows(51);
for (const [ordinal, row] of rows.entries()) {
function activity(
account: string,
payee: string,
visibleBalance: string,
endingBalance: string,
firstNativeIndex: number
) {
const exports = Array.from({ length: 51 }, (_, ordinal) => {
// The latest account date is not the native-final row, and lands alone on page two.
const accountDate = ordinal === 0 ? '2026-04-30' : '2026-04-01';
const source = nativeTransactionFixture(firstNativeIndex + ordinal, [
account,
'expenses:office'
]);
const native = {
...source,
tdescription: `${payee} | supplies`,
tdate: accountDate,
tdate2: null,
tpostings: source.tpostings.map((posting, index) => ({
...posting,
pdate: index === 0 ? accountDate : null,
pdate2: null,
pamount: posting.pamount.map((nativeAmount) => ({ ...nativeAmount, acommodity: '$' }))
}))
};
const originalJson = JSON.stringify(native);
return { originalJson, transactedJson: originalJson };
});
const rows = preparedAccountRowsFromExports(exports);
for (const [ordinal, row] of rows.entries()) {
row.accountDate = ordinal === 0 ? '2026-04-30' : '2026-04-01';
row.isSplit = false;
row.otherAccounts = ['expenses:office'];
row.balance = [amount(visibleBalance)];
row.investment.cash.balance = [amount(visibleBalance)];
row.transaction.payee = payee;
row.transaction.postings = row.transaction.postings.slice(0, 2);
row.transaction.postings[0].account = account;
row.transaction.postings[0].date = row.accountDate;
row.transaction.postings[1].amounts = [amount('100.00')];
row.transaction.postings[1].transactedAmounts = [amount('100.00')];
row.investment.expensePostingIndexes = [1];
}
rows[50].balance = [amount(endingBalance)];
return rows;
return { rows, exports };
}
it('pins selected pages and account summaries until the next request', async () => {
@@ -79,36 +102,40 @@ it('pins selected pages and account summaries until the next request', async ()
cashAccount,
'Market A',
'9007199254740993.123456789',
'8007199254740993.000000001'
'8007199254740993.000000001',
1
);
const cashB = activity(
cashAccount,
'Market B',
'9007199254740994.123456789',
'8007199254740994.000000002'
'8007199254740994.000000002',
1
);
cashB[50].change.push({ ...amount('0.00000002'), commodity: 'EUR' });
cashB.rows[50].change.push({ ...amount('0.00000002'), commodity: 'EUR' });
const investmentA = activity(
investmentAccount,
'Broker A',
'9007199254740993.123456789',
'8007199254740993.000000001'
'8007199254740993.000000001',
52
);
const investmentB = activity(
investmentAccount,
'Broker B',
'9007199254740994.123456789',
'8007199254740994.000000002'
'8007199254740994.000000002',
52
);
investmentA[50].investment.cash.balance = [
investmentA.rows[50].investment.cash.balance = [
amount('10.00000001'),
{ ...amount('-0.00'), commodity: 'EUR' }
];
investmentA[50].investment.cash.unclassifiedBalance = [
investmentA.rows[50].investment.cash.unclassifiedBalance = [
{ ...amount('2.000'), commodity: 'TOKEN' }
];
investmentB[50].investment.cash.balance = [amount('20.00000002')];
investmentB[50].investment.cash.unclassifiedBalance = [
investmentB.rows[50].investment.cash.balance = [amount('20.00000002')];
investmentB.rows[50].investment.cash.unclassifiedBalance = [
{ ...amount('3.000'), commodity: 'OTHER' }
];
@@ -116,8 +143,12 @@ it('pins selected pages and account summaries until the next request', async ()
id: 'A',
createdAt: new Date(overview.preparedAt)
});
await stageA.writeRows(cashAccount, cashA);
await stageA.writeRows(investmentAccount, investmentA);
for (const pair of [...cashA.exports, ...investmentA.exports]) {
await stageA.transactions.insertOriginal(pair.originalJson);
await stageA.transactions.attachTransacted(pair.transactedJson);
}
await stageA.writeRows(cashAccount, cashA.rows);
await stageA.writeRows(investmentAccount, investmentA.rows);
await stageA.publish(
overview,
summaries('111.0000001', '222.0000002'),
@@ -145,8 +176,12 @@ it('pins selected pages and account summaries until the next request', async ()
{ id: 'B', createdAt: new Date(preparedB) },
previous.accounts
);
await stageB.writeRows(cashAccount, cashB);
await stageB.writeRows(investmentAccount, investmentB);
for (const pair of [...cashB.exports, ...investmentB.exports]) {
await stageB.transactions.insertOriginal(pair.originalJson);
await stageB.transactions.attachTransacted(pair.transactedJson);
}
await stageB.writeRows(cashAccount, cashB.rows);
await stageB.writeRows(investmentAccount, investmentB.rows);
await stageB.publish(
{ ...overview, preparedAt: preparedB },
summaries('333.0000003', '444.0000004'),
@@ -192,7 +227,7 @@ it('pins selected pages and account summaries until the next request', async ()
transactionCount: 51,
transactions: [
{
index: 1,
index: 52,
payee: 'Broker A',
cash: { balance: [{ commodity: '$', quantity: '9007199254740993.123456789' }] }
}
@@ -245,7 +280,7 @@ it('pins selected pages and account summaries until the next request', async ()
register: {
transactions: [
{
index: 1,
index: 52,
payee: 'Broker B',
cash: { balance: [{ commodity: '$', quantity: '9007199254740994.123456789' }] }
}
@@ -78,6 +78,30 @@ function expectSavedNativeExports(
.all();
expect(stored).toHaveLength(publication.overview.journalCoverage.transactionCount);
expect(stored.length).toBeGreaterThan(accountRowCount);
const activity = database
.prepare(
`SELECT activity.native_transaction_index, activity.context_json,
transactions.native_index AS canonical_index
FROM account_activity AS activity
LEFT JOIN transactions ON transactions.native_index = activity.native_transaction_index`
)
.all();
expect(activity.length).toBeGreaterThan(0);
for (const record of activity) {
expect(record.canonical_index).toBe(record.native_transaction_index);
const context = JSON.parse(record.context_json as string);
expect(Object.keys(context).sort()).toEqual([
'accountPostingIndexes',
'balance',
'change',
'investment',
'isSplit',
'otherAccounts'
]);
expect(JSON.stringify(context)).not.toMatch(
/"(?:transaction|transactions|postings|tpostings)"\s*:/
);
}
const pairs = stored.map((record) => {
expect(record.original_json).toEqual(expect.any(String));
expect(record.transacted_json).toEqual(expect.any(String));
+3 -1
View File
@@ -4,6 +4,7 @@ import * as reports from './migrations/001_reports.js';
import * as activityIndexes from './migrations/002_account_activity_indexes.js';
import * as activitySummaries from './migrations/003_account_activity_summaries.js';
import * as nativeTransactions from './migrations/004_native_transactions.js';
import * as sharedAccountTransactions from './migrations/005_shared_account_transactions.js';
import type { ReportDatabase } from './schema.js';
export async function runMigrations(database: Kysely<ReportDatabase>): Promise<void> {
@@ -15,7 +16,8 @@ export async function runMigrations(database: Kysely<ReportDatabase>): Promise<v
'001_reports': reports,
'002_account_activity_indexes': activityIndexes,
'003_account_activity_summaries': activitySummaries,
'004_native_transactions': nativeTransactions
'004_native_transactions': nativeTransactions,
'005_shared_account_transactions': sharedAccountTransactions
};
}
}
@@ -0,0 +1,43 @@
import { sql, type Kysely } from 'kysely';
import { up as createActivityIndexes } from './002_account_activity_indexes.js';
/** Fresh builders have no activity rows; published databases are never migrated in place. */
export async function up(database: Kysely<unknown>): Promise<void> {
await database.schema.dropTable('account_activity').execute();
await database.schema
.createTable('account_activity')
.addColumn('account_name', 'text', (column) =>
column.modifyFront(sql`COLLATE BINARY`).notNull()
)
.addColumn('ordinal', 'integer', (column) => column.notNull())
.addColumn('account_date', 'text', (column) => column.notNull())
.addColumn('native_transaction_index', 'integer', (column) => column.notNull())
.addColumn('context_json', 'text', (column) => column.notNull())
.addPrimaryKeyConstraint('account_activity_key', ['account_name', 'ordinal'])
.addUniqueConstraint('account_activity_transaction', [
'account_name',
'native_transaction_index'
])
.addCheckConstraint('account_activity_ordinal', sql`ordinal >= 0`)
.addCheckConstraint(
'account_activity_context_json',
sql`json_valid(context_json) AND json_type(context_json) = 'object'`
)
.addForeignKeyConstraint(
'account_activity_native_transaction',
['native_transaction_index'],
'transactions',
['native_index']
)
.addForeignKeyConstraint(
'account_activity_account',
['account_name'],
'accounts',
['name'],
(constraint) => constraint.deferrable().initiallyDeferred()
)
.modifyEnd(sql`STRICT`)
.execute();
await createActivityIndexes(database);
await sql`PRAGMA user_version = 3`.execute(database);
}
+78 -51
View File
@@ -4,6 +4,12 @@ import type {
PreparedAccountSummary,
PreparedAccountRow
} from '@hledger-reports/hledger';
import {
decodeNativeTransaction,
prepareTransaction,
type NativeTransactionPair
} from '@hledger-reports/hledger/transactions';
import { nativeTransactionFixture } from '../transactions/native.fixture.js';
import { AccountActivitySummaryBuilder } from './account-activity-summary.js';
export function preparedActivitySummary(rows: readonly PreparedAccountRow[] = []) {
@@ -67,13 +73,77 @@ export const amount = (quantity: string): DetailedAmount => ({
style: null
});
export const sourcePosition = { file: '/private/books/included.journal', line: 12, column: 1 };
export function accountTransactionExportsFixture(
count = 1,
description = 'exact producer strings',
firstNativeIndex = 1
): Array<{ originalJson: string; transactedJson: string }> {
return Array.from({ length: count }, (_, i) => {
const native = nativeTransactionFixture(firstNativeIndex + i, [
'assets:cash',
'expenses:食料',
'expenses:household'
]);
const transaction = {
...native,
tdate: '2026-04-29',
tdate2: '2026-04-30',
tstatus: 'Cleared',
tcode: 'REF-42',
tdescription: `${description} ${i}`,
tcomment: 'transaction note\nproject: home\n',
tprecedingcomment: 'retained preceding comment\n',
ttags: [
['project', 'home'],
['project', 'ordered']
],
tsourcepos: [12, 18].map((line) => ({
sourceName: sourcePosition.file,
sourceLine: line,
sourceColumn: 1
})),
tpostings: native.tpostings.map((posting, index) => ({
...posting,
pamount: [
{
acommodity: '$',
aquantity: {
decimalMantissa: [-10000, 7500, 2500][index],
decimalPlaces: 2,
floatingPoint: [-100, 75, 25][index]
},
acost: null,
acostbasis: null,
astyle: null
}
],
pdate: index === 0 ? '2026-04-30' : null,
pdate2: null,
pstatus: 'Unmarked',
pcomment: ['bank note', 'food note', 'household note'][index],
ptags: [['receipt', 'fictional']],
pbalanceassertion: null
}))
};
const json = JSON.stringify(transaction);
return { originalJson: json, transactedJson: json };
});
}
export const preparedAccountRows = (
count = 1,
description = 'exact producer strings'
description = 'exact producer strings',
firstNativeIndex = 1
): PreparedAccountRow[] =>
Array.from(
{ length: count },
(_, i): PreparedAccountRow => ({
preparedAccountRowsFromExports(
accountTransactionExportsFixture(count, description, firstNativeIndex)
);
export const preparedAccountRowsFromExports = (
exports: readonly NativeTransactionPair[]
): PreparedAccountRow[] =>
exports.map(
({ originalJson, transactedJson }): PreparedAccountRow => ({
accountDate: '2026-04-30',
isSplit: true,
otherAccounts: ['expenses:食料', 'expenses:household'],
@@ -95,53 +165,10 @@ export const preparedAccountRows = (
gainPostingIndexes: [],
issues: []
},
transaction: {
index: i + 1,
date: '2026-04-29',
secondaryDate: '2026-04-30',
status: 'Cleared',
code: 'REF-42',
description: `${description} ${i}`,
payee: 'Market',
note: 'supplies',
comment: 'transaction note\nproject: home\n',
precedingComment: 'retained preceding comment\n',
tags: [
['project', 'home'],
['project', 'ordered']
],
postings: [
['assets:cash', '-100.00', 'bank note'],
['expenses:食料', '75.00', 'food note'],
['expenses:household', '25.00', 'household note']
].map(([account, quantity, comment], postingIndex) => {
const date = postingIndex === 0 ? '2026-04-30' : null;
const balanceAssertion =
postingIndex === 0
? {
amount: amount('9007199254740993.123456789'),
total: false,
inclusive: false,
sourcePosition
}
: null;
return {
account,
amounts: [amount(quantity)],
transactedAmounts: [amount(quantity)],
date,
secondaryDate: null,
status: 'Unmarked',
effectiveStatus: 'Cleared',
comment,
tags: [['receipt', 'fictional']],
kind: 'RealPosting',
balanceAssertion,
transactionIndex: String(i + 1)
};
}),
sourcePositions: [sourcePosition, { ...sourcePosition, line: 18 }]
}
transaction: prepareTransaction(
decodeNativeTransaction(originalJson),
decodeNativeTransaction(transactedJson)
)
})
);
export const ready = (name = 'assets:cash', rowCount = 1): PreparedAccountSummary => ({
+36 -19
View File
@@ -95,29 +95,43 @@ export function hasOverviewContainers(value: unknown): value is OverviewData {
);
}
type AccountContext = Pick<
PreparedAccountRow,
'isSplit' | 'otherAccounts' | 'change' | 'balance' | 'accountPostingIndexes' | 'investment'
>;
export function hasAccountContextContainers(value: unknown): value is AccountContext {
return (
isObject(value) &&
typeof value.isSplit === 'boolean' &&
Array.isArray(value.otherAccounts) &&
value.otherAccounts.every((account) => typeof account === 'string') &&
Array.isArray(value.change) &&
Array.isArray(value.balance) &&
Array.isArray(value.accountPostingIndexes) &&
isObject(value.investment) &&
Array.isArray(value.investment.securities) &&
isObject(value.investment.cash) &&
typeof value.investment.cash.direction === 'string' &&
['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) &&
Array.isArray(value.investment.cash.change) &&
Array.isArray(value.investment.cash.balance) &&
Array.isArray(value.investment.cash.unclassifiedChange) &&
Array.isArray(value.investment.cash.unclassifiedBalance) &&
Array.isArray(value.investment.cashPostingIndexes) &&
Array.isArray(value.investment.incomePostingIndexes) &&
Array.isArray(value.investment.expensePostingIndexes) &&
Array.isArray(value.investment.gainPostingIndexes) &&
Array.isArray(value.investment.issues)
);
}
export function hasAccountRowContainers(value: unknown): value is PreparedAccountRow {
if (
!isObject(value) ||
typeof value.accountDate !== 'string' ||
!Array.isArray(value.change) ||
!Array.isArray(value.balance) ||
!isObject(value.transaction) ||
!Array.isArray(value.transaction.postings) ||
!Array.isArray(value.accountPostingIndexes) ||
!isObject(value.investment) ||
!Array.isArray(value.investment.securities) ||
!isObject(value.investment.cash) ||
typeof value.investment.cash.direction !== 'string' ||
!['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) ||
!Array.isArray(value.investment.cash.change) ||
!Array.isArray(value.investment.cash.balance) ||
!Array.isArray(value.investment.cash.unclassifiedChange) ||
!Array.isArray(value.investment.cash.unclassifiedBalance) ||
!Array.isArray(value.investment.cashPostingIndexes) ||
!Array.isArray(value.investment.incomePostingIndexes) ||
!Array.isArray(value.investment.expensePostingIndexes) ||
!Array.isArray(value.investment.gainPostingIndexes) ||
!Array.isArray(value.investment.issues) ||
!value.transaction.postings.every(
(posting) =>
isObject(posting) &&
@@ -126,8 +140,11 @@ export function hasAccountRowContainers(value: unknown): value is PreparedAccoun
)
return false;
const postingCount = value.transaction.postings.length;
return value.accountPostingIndexes.every(
(index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount
return (
hasAccountContextContainers(value) &&
value.accountPostingIndexes.every(
(index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount
)
);
}
+220 -51
View File
@@ -17,14 +17,19 @@ import {
writeAccountActivitySummaries,
writeReportData
} from './queries.js';
import type { ReportTransaction } from '../database.js';
import { insertNativeTransaction, attachTransactedTransaction } from '../transactions/index.js';
import { richNativeExportsFixture } from '../transactions/native.fixture.js';
import type { StoredReportData } from './data.js';
import {
amount,
accountTransactionExportsFixture,
ambiguous,
preparedOverview,
journal,
ready,
preparedAccountRows,
preparedAccountRowsFromExports,
preparedActivitySummary
} from './data.fixture.js';
@@ -57,43 +62,49 @@ afterEach(async () => {
}
});
async function importTransactions(
transaction: ReportTransaction,
exports: Array<{ originalJson: string; transactedJson: string }>
): Promise<void> {
for (const { originalJson, transactedJson } of exports) {
await insertNativeTransaction(transaction, originalJson);
await attachTransactedTransaction(transaction, transactedJson);
}
}
function caseDistinctExports(firstNativeIndex: number) {
return accountTransactionExportsFixture(1, 'case-distinct account', firstNativeIndex).map(
({ originalJson }) => {
const native = JSON.parse(originalJson);
native.tpostings[0].paccount = 'assets:Cash';
const json = JSON.stringify(native);
return { originalJson: json, transactedJson: json };
}
);
}
it('stores exact prepared payloads and native row order using the migrated schema', async () => {
const database = await createDatabase();
const rows = preparedAccountRows(52).reverse();
const rich = rows[0];
// Preserve producer-owned prices and basis, including unavailable native conversions.
rich.transaction.postings[1].amounts[0] = {
...amount('75.00'),
commodity: 'TEST',
cost: { kind: 'UnitCost', amount: amount('0.000000000000000000012300') },
costBasis: {
amount: { ...amount('9007199254740993.123456789'), commodity: 'EUR' },
date: '2026-04-01',
label: 'retained basis label'
},
style: {
commoditySide: 'L',
commoditySpaced: false,
decimalMark: '.',
digitGroups: [',', [3]],
precision: 2,
rounding: 'NoRounding'
}
};
rich.transaction.postings[1].transactedAmounts = [
{ commodity: 'EUR', quantity: '0.000000000000000000922500' }
const richExports = richNativeExportsFixture();
richExports.transactedJson = richExports.transactedJson.replace(
'"decimalMantissa": 10000',
'"decimalMantissa": 9007199254740993123456789'
);
const exports = [
richExports,
...accountTransactionExportsFixture(51, 'exact producer strings', 8).reverse()
];
rich.transaction.postings[2].tags.push(['_conversion-posting', '']);
rich.transaction.postings[2].transactedAmounts = null;
const rows = preparedAccountRowsFromExports(exports);
const rich = rows[0];
rich.investment = {
securities: [
{
commodity: 'TEST',
commodity: 'USD',
action: 'unknown',
change: [{ commodity: 'TEST', quantity: '75.00' }],
postingIndexes: [1],
prices: [amount('0.000000000000000000012300')],
transactedAmounts: rich.transaction.postings[1].transactedAmounts
change: [{ commodity: 'USD', quantity: rich.transaction.postings[0].amounts[0].quantity }],
postingIndexes: [0],
prices: [rich.transaction.postings[0].amounts[0].cost!.amount],
transactedAmounts: rich.transaction.postings[0].transactedAmounts
}
],
cash: {
@@ -117,8 +128,8 @@ it('stores exact prepared payloads and native row order using the migrated schem
];
nativeEnding.investment.cash.balance = [];
nativeEnding.investment.cash.unclassifiedBalance = [{ commodity: 'EUR', quantity: '0.000' }];
const otherRows = preparedAccountRows(1, 'case-distinct account');
otherRows[0].transaction.postings[0].account = 'assets:Cash';
const otherExports = caseDistinctExports(60);
const otherRows = preparedAccountRowsFromExports(otherExports);
const data: StoredReportData = {
journal,
overview: preparedOverview(),
@@ -136,6 +147,7 @@ it('stores exact prepared payloads and native row order using the migrated schem
]);
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, [...exports, ...otherExports]);
await writeAccountRows(transaction, 'assets:cash', 0, rows.slice(0, 51));
await writeAccountRows(transaction, 'assets:cash', 51, rows.slice(51));
await writeAccountRows(transaction, 'assets:Cash', 0, otherRows);
@@ -163,6 +175,9 @@ it('stores exact prepared payloads and native row order using the migrated schem
offset: 50
});
expect(first).toEqual({ rows: rows.slice(0, 50), summary: summaries.get('assets:cash') });
expect(first?.rows[0].transaction.postings[0].transactedAmounts).toEqual([
{ commodity: 'USD', quantity: '90071992547409931234567.89' }
]);
expect(last).toEqual({ rows: rows.slice(50), summary: first?.summary });
expect(await readAccountActivityPage(reader, data.accounts[2], firstPage)).toEqual({
rows: otherRows,
@@ -175,18 +190,18 @@ it('stores exact prepared payloads and native row order using the migrated schem
expect((await stat(path)).mode & 0o777).toBe(0o600);
});
it('reads date-ordered pages with native ties and exact account scope', async () => {
it('shares one canonical transaction without embedding its body in account context', async () => {
const database = await createDatabase();
const rows = preparedAccountRows(53).reverse();
for (const [ordinal, row] of rows.entries()) {
row.accountDate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
row.transaction.postings[0].date = row.accountDate;
}
const otherRows = preparedAccountRows(1, 'case-distinct account');
otherRows[0].transaction.postings[0].account = 'assets:Cash';
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
const other = { id: 'cash0002', ...ready('assets:Cash', otherRows.length) };
await database.transaction().execute(async (transaction) => {
const exports = accountTransactionExportsFixture();
const rows = preparedAccountRowsFromExports(exports);
const otherRows = preparedAccountRowsFromExports(exports);
otherRows[0].accountPostingIndexes = [1];
otherRows[0].otherAccounts = ['assets:cash', 'expenses:household'];
const account = { id: 'cash0001', ...ready('assets:cash') };
const other = { id: 'food0001', ...ready('expenses:食料') };
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, exports);
await writeAccountRows(transaction, account.name, 0, rows);
await writeAccountRows(transaction, other.name, 0, otherRows);
await writeReportData(transaction, {
@@ -201,7 +216,82 @@ it('reads date-ordered pages with native ties and exact account scope', async ()
[other.name, preparedActivitySummary(otherRows)]
])
);
});
await transaction.commit().execute();
} finally {
if (!transaction.isCommitted && !transaction.isRolledBack)
await transaction.rollback().execute();
}
expect(await database.selectFrom('transactions').select('native_index').execute()).toEqual([
{ native_index: 1 }
]);
const stored = await database
.selectFrom('account_activity')
.selectAll()
.orderBy('account_name')
.execute();
expect(stored.map((row) => row.native_transaction_index)).toEqual([1, 1]);
for (const [ordinal, row] of stored.entries()) {
const expected = ordinal === 0 ? rows[0] : otherRows[0];
expect(JSON.parse(row.context_json)).toEqual({
isSplit: expected.isSplit,
otherAccounts: expected.otherAccounts,
change: expected.change,
balance: expected.balance,
accountPostingIndexes: expected.accountPostingIndexes,
investment: expected.investment
});
}
const page = { order: 'asc', offset: 0, limit: 50 } as const;
expect((await readAccountActivityPage(database, account, page))?.rows).toEqual(rows);
expect((await readAccountActivityPage(database, other, page))?.rows).toEqual(otherRows);
// An account has one activity row per canonical transaction, not one per posting.
await expect(writeAccountRows(database, account.name, 1, rows)).rejects.toThrow();
await expect(
writeAccountRows(database, account.name, 1, preparedAccountRows(1, 'not imported', 99))
).rejects.toThrow();
});
it('reads date-ordered pages with native ties and exact account scope', async () => {
const database = await createDatabase();
const exports = accountTransactionExportsFixture(53)
.reverse()
.map(({ originalJson }, ordinal) => {
const native = JSON.parse(originalJson);
native.tpostings[0].pdate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
const json = JSON.stringify(native);
return { originalJson: json, transactedJson: json };
});
const rows = preparedAccountRowsFromExports(exports);
for (const [ordinal, row] of rows.entries()) {
row.accountDate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
}
const otherExports = caseDistinctExports(54);
const otherRows = preparedAccountRowsFromExports(otherExports);
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
const other = { id: 'cash0002', ...ready('assets:Cash', otherRows.length) };
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, [...exports, ...otherExports]);
await writeAccountRows(transaction, account.name, 0, rows);
await writeAccountRows(transaction, other.name, 0, otherRows);
await writeReportData(transaction, {
journal,
overview: preparedOverview(),
accounts: [account, other]
});
await writeAccountActivitySummaries(
transaction,
new Map([
[account.name, preparedActivitySummary(rows)],
[other.name, preparedActivitySummary(otherRows)]
])
);
await transaction.commit().execute();
} finally {
if (!transaction.isCommitted && !transaction.isRolledBack)
await transaction.rollback().execute();
}
const resultRowCounts: number[] = [];
const pageReader = database.withPlugin({
transformQuery: ({ node }) => node,
@@ -245,22 +335,39 @@ it('reads date-ordered pages with native ties and exact account scope', async ()
it('decodes only the requested page and rejects invalid posting references', async () => {
const database = await createDatabase();
const account = { id: 'cash0001', ...ready('assets:cash', 2) };
const rows = preparedAccountRows(2);
const exports = accountTransactionExportsFixture(2);
const rows = preparedAccountRowsFromExports(exports);
await writeReportData(database, { journal, overview: preparedOverview(), accounts: [account] });
await writeAccountActivitySummaries(
database,
new Map([[account.name, preparedActivitySummary(rows)]])
);
await writeAccountRows(database, account.name, 0, rows);
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, exports);
await writeAccountRows(transaction, account.name, 0, rows);
await transaction.commit().execute();
} finally {
if (!transaction.isCommitted && !transaction.isRolledBack)
await transaction.rollback().execute();
}
await closeDatabase(database);
// Valid SQLite/JSON is not sufficient: saved posting references must still be usable.
const native = new DatabaseSync(path);
try {
const badRow = { ...rows[1], accountPostingIndexes: [rows[1].transaction.postings.length] };
const saved = native
.prepare('SELECT context_json FROM account_activity WHERE ordinal = 1')
.get();
const badContext = {
...JSON.parse(String(saved!.context_json)),
accountPostingIndexes: [rows[1].transaction.postings.length]
};
native
.prepare('UPDATE account_activity SET row_json = ? WHERE account_name = ? AND ordinal = 1')
.run(JSON.stringify(badRow), account.name);
.prepare(
'UPDATE account_activity SET context_json = ? WHERE account_name = ? AND ordinal = 1'
)
.run(JSON.stringify(badContext), account.name);
} finally {
native.close();
}
@@ -276,12 +383,74 @@ it('decodes only the requested page and rejects invalid posting references', asy
).rejects.toThrow();
});
it('rejects selected missing or invalid canonical exports without decoding off-page transactions', async () => {
const database = await createDatabase();
const exports = accountTransactionExportsFixture(4);
const rows = preparedAccountRowsFromExports(exports);
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, exports);
await writeAccountRows(transaction, account.name, 0, rows);
await writeReportData(transaction, {
journal,
overview: preparedOverview(),
accounts: [account]
});
await writeAccountActivitySummaries(
transaction,
new Map([[account.name, preparedActivitySummary(rows)]])
);
await transaction.commit().execute();
} finally {
if (!transaction.isCommitted && !transaction.isRolledBack)
await transaction.rollback().execute();
}
await closeDatabase(database);
const native = new DatabaseSync(path);
try {
// Simulate damaged publication storage, including an orphan that normal writes forbid.
native.exec('PRAGMA foreign_keys = OFF');
const wrongIdentity = accountTransactionExportsFixture(1, 'wrong canonical identity', 99)[0];
native
.prepare(
'UPDATE transactions SET original_json = ?, transacted_json = ? WHERE native_index = 2'
)
.run(wrongIdentity.originalJson, wrongIdentity.transactedJson);
native.prepare('UPDATE transactions SET transacted_json = NULL WHERE native_index = 3').run();
native.prepare('DELETE FROM transactions WHERE native_index = 1').run();
} finally {
native.close();
}
const reader = await openReadOnlyReportDatabase(path);
if (!reader) throw new Error('Expected the current report schema.');
databases.add(reader);
expect((await readReportData(reader)).accounts).toEqual([account]);
const readRow = (offset: number) =>
readAccountActivityPage(reader, account, { order: 'asc', offset, limit: 1 });
// The orphan still occupies its ordinal; a join must not shift later rows into the page.
expect(await readRow(3)).toEqual({ rows: rows.slice(3), summary: preparedActivitySummary(rows) });
await expect(readRow(0)).rejects.toThrow();
await expect(readRow(1)).rejects.toThrow();
await expect(readRow(2)).rejects.toThrow();
});
it('rejects missing summaries and malformed native bags rather than inferring from a page', async () => {
const database = await createDatabase();
const account = { id: 'cash0001', ...ready() };
const rows = preparedAccountRows();
const exports = accountTransactionExportsFixture();
const rows = preparedAccountRowsFromExports(exports);
await writeReportData(database, { journal, overview: preparedOverview(), accounts: [account] });
await writeAccountRows(database, account.name, 0, rows);
const transaction = await database.startTransaction().execute();
try {
await importTransactions(transaction, exports);
await writeAccountRows(transaction, account.name, 0, rows);
await transaction.commit().execute();
} finally {
if (!transaction.isCommitted && !transaction.isRolledBack)
await transaction.rollback().execute();
}
const page = { order: 'asc', offset: 0, limit: 50 } as const;
await expect(readAccountActivityPage(database, account, page)).rejects.toThrow();
await expect(writeAccountActivitySummaries(database, new Map())).rejects.toThrow();
+65 -11
View File
@@ -1,4 +1,5 @@
import type { PreparedAccountRow } from '@hledger-reports/hledger';
import { decodeNativeTransaction, prepareTransaction } from '@hledger-reports/hledger/transactions';
import type { Kysely } from 'kysely';
import type { ReportDatabase } from '../schema.js';
import {
@@ -7,6 +8,7 @@ import {
type AccountActivitySummary
} from './account-activity-summary.js';
import {
hasAccountContextContainers,
hasAccountRowContainers,
hasStoredReportData,
type StoredAccount,
@@ -29,8 +31,12 @@ export interface AccountActivityPageOptions {
type StoredActivityRow = Pick<
ReportDatabase['account_activity'],
'ordinal' | 'account_date' | 'row_json'
>;
'ordinal' | 'account_date' | 'native_transaction_index' | 'context_json'
> & {
canonical_native_index: number | null;
original_json: string | null;
transacted_json: string | null;
};
export async function writeAccountRows(
database: Kysely<ReportDatabase>,
@@ -42,12 +48,25 @@ export async function writeAccountRows(
throw new Error('Invalid account rows.');
for (let start = 0; start < rows.length; start += insertBatchSize) {
const batch = rows.slice(start, start + insertBatchSize).map((row, index) => {
if (!hasAccountRowContainers(row)) throw new Error('Invalid account rows.');
if (
!hasAccountRowContainers(row) ||
!Number.isSafeInteger(row.transaction.index) ||
row.transaction.index <= 0
)
throw new Error('Invalid account rows.');
return {
account_name: name,
ordinal: firstOrdinal + start + index,
account_date: row.accountDate,
row_json: JSON.stringify(row)
native_transaction_index: row.transaction.index,
context_json: JSON.stringify({
isSplit: row.isSplit,
otherAccounts: row.otherAccounts,
change: row.change,
balance: row.balance,
accountPostingIndexes: row.accountPostingIndexes,
investment: row.investment
})
};
});
await database.insertInto('account_activity').values(batch).execute();
@@ -143,15 +162,37 @@ export async function readReportData(database: Kysely<ReportDatabase>): Promise<
}
function decodeAccountRow(record: StoredActivityRow, rowCount: number): PreparedAccountRow {
const row = decodePayload(record.row_json);
if (
!Number.isSafeInteger(record.ordinal) ||
record.ordinal < 0 ||
record.ordinal >= rowCount ||
!hasAccountRowContainers(row) ||
record.account_date !== row.accountDate
!Number.isSafeInteger(record.native_transaction_index) ||
record.native_transaction_index <= 0 ||
record.canonical_native_index !== record.native_transaction_index ||
typeof record.original_json !== 'string' ||
typeof record.transacted_json !== 'string'
)
throw new Error('Invalid account data.');
const context = decodePayload(record.context_json);
if (!hasAccountContextContainers(context)) throw new Error('Invalid account data.');
const original = decodeNativeTransaction(record.original_json);
const transacted = decodeNativeTransaction(record.transacted_json);
if (
original.tindex !== record.native_transaction_index ||
transacted.tindex !== record.native_transaction_index
)
throw new Error('Invalid account data.');
const row = {
accountDate: record.account_date,
isSplit: context.isSplit,
otherAccounts: context.otherAccounts,
change: context.change,
balance: context.balance,
accountPostingIndexes: context.accountPostingIndexes,
investment: context.investment,
transaction: prepareTransaction(original, transacted)
};
if (!hasAccountRowContainers(row)) throw new Error('Invalid account data.');
return row;
}
@@ -233,10 +274,23 @@ export async function readAccountActivityPage(
const summary = decodeActivitySummary(storedSummary);
const records = await database
.selectFrom('account_activity')
.select(['ordinal', 'account_date', 'row_json'])
.where('account_name', '=', account.name)
.orderBy('account_date', order)
.orderBy('ordinal', 'asc')
.leftJoin(
'transactions',
'transactions.native_index',
'account_activity.native_transaction_index'
)
.select([
'account_activity.ordinal',
'account_activity.account_date',
'account_activity.native_transaction_index',
'account_activity.context_json',
'transactions.native_index as canonical_native_index',
'transactions.original_json',
'transactions.transacted_json'
])
.where('account_activity.account_name', '=', account.name)
.orderBy('account_activity.account_date', order)
.orderBy('account_activity.ordinal', 'asc')
.limit(limit)
.offset(offset)
.execute();
+3 -2
View File
@@ -1,4 +1,4 @@
export const reportSchemaVersion = 2;
export const reportSchemaVersion = 3;
interface PublicationTable {
singleton: 1;
@@ -20,7 +20,8 @@ interface AccountActivityTable {
account_name: string;
ordinal: number;
account_date: string;
row_json: string;
native_transaction_index: number;
context_json: string;
}
interface AccountActivitySummaryTable {
+1 -1
View File
@@ -1,4 +1,4 @@
export { decodeNativeTransaction } from './export.js';
export { assertTransactionExportsMatch } from './transaction.js';
export { assertTransactionExportsMatch, prepareTransaction } from './transaction.js';
export type { NativeTransaction } from './transaction.js';
export type { NativeTransactionPair, PreparationTransactions } from './store.js';
@@ -13,9 +13,10 @@ import {
readOverview,
readReportPublication
} from '../index.js';
import type { ReportPublication, StoredAccount } from '../index.js';
import type { ReportPublication, ReportStage, StoredAccount } from '../index.js';
import {
ambiguous,
accountTransactionExportsFixture,
preparedOverview as fixture,
journal,
ready,
@@ -46,12 +47,25 @@ const newer = { id: 'newer', createdAt: new Date('2026-05-01T00:00:00.000Z') };
const safeError = (message: string) => (error: Error) =>
error.message === message && error.cause === undefined;
async function importNativeExports(
stage: ReportStage,
count = 1,
description = 'exact producer strings',
firstIndex = 1
) {
for (const pair of accountTransactionExportsFixture(count, description, firstIndex)) {
await stage.transactions.insertOriginal(pair.originalJson);
await stage.transactions.attachTransacted(pair.transactedJson);
}
}
async function publish(
run = older,
data = fixture(),
previousAccounts?: readonly StoredAccount[] | null
) {
const stage = await createReportStage(snapshots, run, previousAccounts);
await importNativeExports(stage);
await stage.writeRows('assets:cash', rows());
return stage.publish(data, [ready()], journal);
}
@@ -94,6 +108,7 @@ it('commits and closes a complete private database before making its directory v
return database;
});
const stage = await createReportStage(snapshots, newer, pinned.accounts);
await importNativeExports(stage, 52);
const transactions = rows(52);
transactions[0].change.push({
...transactions[0].change[0],
@@ -122,10 +137,22 @@ it('commits and closes a complete private database before making its directory v
expect(JSON.parse(record.overview_json as string)).toEqual(fixture());
expect(
database
.prepare('SELECT row_json FROM account_activity ORDER BY ordinal')
.prepare(
'SELECT account_date, native_transaction_index, context_json FROM account_activity ORDER BY ordinal'
)
.all()
.map((row) => JSON.parse(row.row_json as string))
).toEqual(transactions);
.map((row) => ({
accountDate: row.account_date,
nativeIndex: row.native_transaction_index,
context: JSON.parse(row.context_json as string)
}))
).toEqual(
transactions.map(({ accountDate, transaction, ...context }) => ({
accountDate,
nativeIndex: transaction.index,
context
}))
);
} finally {
database.close();
}
@@ -297,6 +324,7 @@ it('reuses IDs by exact account identity and avoids collisions with removed and
generated.push(0, 1, 2, 2, 3);
const stage = await createReportStage(snapshots, newer, previous.accounts);
const names = ['資産/../銀行:Ω', 'new:Ω', 'NEW:Ω'];
await importNativeExports(stage);
for (const name of names) await stage.writeRows(name, rows());
await stage.publish(
fixture(),
@@ -323,6 +351,7 @@ it('keeps completed generations pinned through same-request recovery and discard
await publish();
const pinned = await selected();
const stage = await createReportStage(snapshots, older, pinned.accounts);
await importNativeExports(stage, 1, 'recovered');
await stage.writeRows('assets:cash', rows(1, 'recovered'));
expect(await selected()).toEqual(pinned);
await stage.publish({ ...fixture(), preparedAt: 'recovered' }, [ready()], journal);
@@ -517,6 +546,7 @@ it.each([
await publish();
const pinned = await selected();
const stage = await createReportStage(snapshots, newer, pinned.accounts);
await importNativeExports(stage);
await stage.writeRows('assets:cash', rows());
await expect(stage.publish(fixture(), [summary], journal)).rejects.toSatisfy(
safeError('Unable to prepare reports')
@@ -531,6 +561,7 @@ it('cleans only its stage when the publication directory rename fails', async ()
const sets = join(snapshots, '.sets');
await fs.mkdir(join(sets, '.foreign.tmp'));
const stage = await createReportStage(snapshots, newer, pinned.accounts);
await importNativeExports(stage);
await stage.writeRows('assets:cash', rows());
vi.spyOn(fs, 'rename').mockRejectedValue(new Error(`${snapshots}: private rename failure`));
await expect(stage.publish(fixture(), [ready()], journal)).rejects.toSatisfy(
@@ -556,6 +587,7 @@ it('rejects noncontiguous writes and retries removal without closing the databas
return database;
});
const stage = await createReportStage(snapshots, older);
await importNativeExports(stage);
await stage.writeRows('a', rows());
await stage.writeRows('b', rows());
const remove = fs.rm;
@@ -592,6 +624,7 @@ it('publishes without financial readback and keeps the new generation after alia
.spyOn(databases, 'openReadOnlyReportDatabase')
.mockRejectedValue(new Error('Publication must not read selected financial data'));
const stage = await createReportStage(snapshots, newer, pinned.accounts);
await importNativeExports(stage, 1, 'committed');
await stage.writeRows('assets:cash', rows(1, 'committed'));
const rename = fs.rename;
vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => {
@@ -702,14 +735,14 @@ it('reports a bad selected database without falling back or changing its bytes',
expect(previousPage?.rows).toEqual(rows());
});
it('withholds a selected v1 SQLite publication until explicit preparation without changing its bytes', async () => {
it('withholds a selected v2 SQLite publication until explicit preparation without changing its bytes', async () => {
await publish();
const previous = await selected();
await publish(newer);
const path = join((await selected()).directory, 'reports.sqlite');
const database = new DatabaseSync(path);
try {
database.exec('PRAGMA user_version = 1');
database.exec('PRAGMA user_version = 2');
} finally {
database.close();
}
@@ -728,25 +761,21 @@ it('withholds a selected v1 SQLite publication until explicit preparation withou
expect(await fs.readFile(path)).toEqual(bytes);
});
it('leaves a selected JSON-only generation unavailable and untouched until explicit preparation', async () => {
it('requires selected SQLite even when an old financial JSON file exists', async () => {
await publish();
const previous = await selected();
const token = `${String(newer.createdAt.getTime()).padStart(16, '0')}-${newer.id}-00000000-0000-4000-8000-000000000000`;
const directory = join(snapshots, '.sets', token);
await fs.mkdir(join(directory, 'transactions'), { recursive: true, mode: 0o700 });
const overviewPath = join(directory, 'overview.json');
const accountPath = join(directory, 'transactions', `${previous.accounts[0].id}.json`);
const overviewBytes = JSON.stringify({
format: 8,
journal,
overview: fixture(),
accounts: previous.accounts
});
const accountBytes = 'opaque legacy account payload';
await fs.writeFile(overviewPath, overviewBytes);
await fs.writeFile(accountPath, accountBytes);
expect(await readReportPublication(snapshots)).toBeNull();
expect(await readOverview(snapshots)).toBeNull();
await fs.mkdir(directory, { mode: 0o700 });
const jsonPath = join(directory, 'overview.json');
const bytes = Buffer.from('{"obsolete":"fictional private report"}');
await fs.writeFile(jsonPath, bytes);
await expect(readReportPublication(snapshots)).rejects.toSatisfy(
safeError('Unable to read overview')
);
await expect(readOverview(snapshots)).rejects.toSatisfy(safeError('Unable to read overview'));
expect(await fs.readFile(jsonPath)).toEqual(bytes);
expect(await fs.readdir(directory)).toEqual(['overview.json']);
const previousPage = await readAccountPage(previous, previous.accounts[0].id, {
order: 'asc',
offset: 0,
@@ -755,9 +784,8 @@ it('leaves a selected JSON-only generation unavailable and untouched until expli
expect(previousPage?.rows).toEqual(rows());
await publish({ id: 'explicit', createdAt: new Date(newer.createdAt.getTime() + 1) });
expect(await readOverview(snapshots)).toEqual(fixture());
expect(await fs.readFile(overviewPath, 'utf8')).toBe(overviewBytes);
expect(await fs.readFile(accountPath, 'utf8')).toBe(accountBytes);
expect((await fs.readdir(directory)).sort()).toEqual(['overview.json', 'transactions']);
expect(await fs.readFile(jsonPath)).toEqual(bytes);
expect(await fs.readdir(directory)).toEqual(['overview.json']);
});
it('rejects a symlink in place of the selected completed directory', async () => {
+2 -13
View File
@@ -40,7 +40,7 @@ import {
type StoredNativeTransaction
} from '@hledger-reports/db/transactions';
import { randomInt, randomUUID } from 'node:crypto';
import { lstat, mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
import { mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
import { basename, dirname, join } from 'node:path';
import { setImmediate } from 'node:timers/promises';
import { namesIn, privateDirectory } from '../files.js';
@@ -88,17 +88,6 @@ async function latestSet(snapshotDirectory: string): Promise<string | undefined>
.at(-1);
}
async function openSelectedDatabase(directory: string): Promise<ReportConnection | null> {
try {
return await openReadOnlyReportDatabase(join(directory, databaseName));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
const legacy = await lstat(join(directory, 'overview.json'));
if (!legacy.isFile()) throw error;
return null;
}
}
export async function readReportPublication(
snapshotDirectory: string
): Promise<ReportPublication | null> {
@@ -107,7 +96,7 @@ export async function readReportPublication(
if (!latest) return null;
const directory = join(snapshotDirectory, '.sets', latest);
await privateDirectory(directory);
const database = await openSelectedDatabase(directory);
const database = await openReadOnlyReportDatabase(join(directory, databaseName));
if (!database) return null;
try {
const data = await readReportData(database);