Read shared native transactions in saved registers #51
No files matched your search
@@ -0,0 +1,6 @@
|
||||
---
|
||||
'@hledger-reports/web': patch
|
||||
'@hledger-reports/worker': patch
|
||||
---
|
||||
|
||||
Read saved registers from shared native transactions instead of per-account copies and remove legacy financial JSON-file recognition. Prepare reports again after updating.
|
||||
@@ -6,31 +6,54 @@ import type { PreparedAccountSummary } from '@hledger-reports/hledger';
|
||||
import { createReportStage, readReportPublication } from '@hledger-reports/snapshots';
|
||||
import {
|
||||
amount,
|
||||
preparedAccountRows,
|
||||
preparedAccountRowsFromExports,
|
||||
preparedOverview
|
||||
} from '../../../../../packages/db/src/reports/data.fixture.js';
|
||||
import { nativeTransactionFixture } from '../../../../../packages/db/src/transactions/native.fixture.js';
|
||||
import { load as loadLayout } from '../../routes/(protected)/+layout.server.js';
|
||||
import { load as loadCashAccount } from '../../routes/(protected)/registers/cash/[accountId]/+page.server.js';
|
||||
import { load as loadInvestmentAccount } from '../../routes/(protected)/registers/investments/[accountId]/+page.server.js';
|
||||
|
||||
function activity(account: string, payee: string, visibleBalance: string, endingBalance: string) {
|
||||
const rows = preparedAccountRows(51);
|
||||
for (const [ordinal, row] of rows.entries()) {
|
||||
function activity(
|
||||
account: string,
|
||||
payee: string,
|
||||
visibleBalance: string,
|
||||
endingBalance: string,
|
||||
firstNativeIndex: number
|
||||
) {
|
||||
const exports = Array.from({ length: 51 }, (_, ordinal) => {
|
||||
// The latest account date is not the native-final row, and lands alone on page two.
|
||||
const accountDate = ordinal === 0 ? '2026-04-30' : '2026-04-01';
|
||||
const source = nativeTransactionFixture(firstNativeIndex + ordinal, [
|
||||
account,
|
||||
'expenses:office'
|
||||
]);
|
||||
const native = {
|
||||
...source,
|
||||
tdescription: `${payee} | supplies`,
|
||||
tdate: accountDate,
|
||||
tdate2: null,
|
||||
tpostings: source.tpostings.map((posting, index) => ({
|
||||
...posting,
|
||||
pdate: index === 0 ? accountDate : null,
|
||||
pdate2: null,
|
||||
pamount: posting.pamount.map((nativeAmount) => ({ ...nativeAmount, acommodity: '$' }))
|
||||
}))
|
||||
};
|
||||
const originalJson = JSON.stringify(native);
|
||||
return { originalJson, transactedJson: originalJson };
|
||||
});
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
for (const [ordinal, row] of rows.entries()) {
|
||||
row.accountDate = ordinal === 0 ? '2026-04-30' : '2026-04-01';
|
||||
row.isSplit = false;
|
||||
row.otherAccounts = ['expenses:office'];
|
||||
row.balance = [amount(visibleBalance)];
|
||||
row.investment.cash.balance = [amount(visibleBalance)];
|
||||
row.transaction.payee = payee;
|
||||
row.transaction.postings = row.transaction.postings.slice(0, 2);
|
||||
row.transaction.postings[0].account = account;
|
||||
row.transaction.postings[0].date = row.accountDate;
|
||||
row.transaction.postings[1].amounts = [amount('100.00')];
|
||||
row.transaction.postings[1].transactedAmounts = [amount('100.00')];
|
||||
row.investment.expensePostingIndexes = [1];
|
||||
}
|
||||
rows[50].balance = [amount(endingBalance)];
|
||||
return rows;
|
||||
return { rows, exports };
|
||||
}
|
||||
|
||||
it('pins selected pages and account summaries until the next request', async () => {
|
||||
@@ -79,36 +102,40 @@ it('pins selected pages and account summaries until the next request', async ()
|
||||
cashAccount,
|
||||
'Market A',
|
||||
'9007199254740993.123456789',
|
||||
'8007199254740993.000000001'
|
||||
'8007199254740993.000000001',
|
||||
1
|
||||
);
|
||||
const cashB = activity(
|
||||
cashAccount,
|
||||
'Market B',
|
||||
'9007199254740994.123456789',
|
||||
'8007199254740994.000000002'
|
||||
'8007199254740994.000000002',
|
||||
1
|
||||
);
|
||||
cashB[50].change.push({ ...amount('0.00000002'), commodity: 'EUR' });
|
||||
cashB.rows[50].change.push({ ...amount('0.00000002'), commodity: 'EUR' });
|
||||
const investmentA = activity(
|
||||
investmentAccount,
|
||||
'Broker A',
|
||||
'9007199254740993.123456789',
|
||||
'8007199254740993.000000001'
|
||||
'8007199254740993.000000001',
|
||||
52
|
||||
);
|
||||
const investmentB = activity(
|
||||
investmentAccount,
|
||||
'Broker B',
|
||||
'9007199254740994.123456789',
|
||||
'8007199254740994.000000002'
|
||||
'8007199254740994.000000002',
|
||||
52
|
||||
);
|
||||
investmentA[50].investment.cash.balance = [
|
||||
investmentA.rows[50].investment.cash.balance = [
|
||||
amount('10.00000001'),
|
||||
{ ...amount('-0.00'), commodity: 'EUR' }
|
||||
];
|
||||
investmentA[50].investment.cash.unclassifiedBalance = [
|
||||
investmentA.rows[50].investment.cash.unclassifiedBalance = [
|
||||
{ ...amount('2.000'), commodity: 'TOKEN' }
|
||||
];
|
||||
investmentB[50].investment.cash.balance = [amount('20.00000002')];
|
||||
investmentB[50].investment.cash.unclassifiedBalance = [
|
||||
investmentB.rows[50].investment.cash.balance = [amount('20.00000002')];
|
||||
investmentB.rows[50].investment.cash.unclassifiedBalance = [
|
||||
{ ...amount('3.000'), commodity: 'OTHER' }
|
||||
];
|
||||
|
||||
@@ -116,8 +143,12 @@ it('pins selected pages and account summaries until the next request', async ()
|
||||
id: 'A',
|
||||
createdAt: new Date(overview.preparedAt)
|
||||
});
|
||||
await stageA.writeRows(cashAccount, cashA);
|
||||
await stageA.writeRows(investmentAccount, investmentA);
|
||||
for (const pair of [...cashA.exports, ...investmentA.exports]) {
|
||||
await stageA.transactions.insertOriginal(pair.originalJson);
|
||||
await stageA.transactions.attachTransacted(pair.transactedJson);
|
||||
}
|
||||
await stageA.writeRows(cashAccount, cashA.rows);
|
||||
await stageA.writeRows(investmentAccount, investmentA.rows);
|
||||
await stageA.publish(
|
||||
overview,
|
||||
summaries('111.0000001', '222.0000002'),
|
||||
@@ -145,8 +176,12 @@ it('pins selected pages and account summaries until the next request', async ()
|
||||
{ id: 'B', createdAt: new Date(preparedB) },
|
||||
previous.accounts
|
||||
);
|
||||
await stageB.writeRows(cashAccount, cashB);
|
||||
await stageB.writeRows(investmentAccount, investmentB);
|
||||
for (const pair of [...cashB.exports, ...investmentB.exports]) {
|
||||
await stageB.transactions.insertOriginal(pair.originalJson);
|
||||
await stageB.transactions.attachTransacted(pair.transactedJson);
|
||||
}
|
||||
await stageB.writeRows(cashAccount, cashB.rows);
|
||||
await stageB.writeRows(investmentAccount, investmentB.rows);
|
||||
await stageB.publish(
|
||||
{ ...overview, preparedAt: preparedB },
|
||||
summaries('333.0000003', '444.0000004'),
|
||||
@@ -192,7 +227,7 @@ it('pins selected pages and account summaries until the next request', async ()
|
||||
transactionCount: 51,
|
||||
transactions: [
|
||||
{
|
||||
index: 1,
|
||||
index: 52,
|
||||
payee: 'Broker A',
|
||||
cash: { balance: [{ commodity: '$', quantity: '9007199254740993.123456789' }] }
|
||||
}
|
||||
@@ -245,7 +280,7 @@ it('pins selected pages and account summaries until the next request', async ()
|
||||
register: {
|
||||
transactions: [
|
||||
{
|
||||
index: 1,
|
||||
index: 52,
|
||||
payee: 'Broker B',
|
||||
cash: { balance: [{ commodity: '$', quantity: '9007199254740994.123456789' }] }
|
||||
}
|
||||
|
||||
@@ -78,6 +78,30 @@ function expectSavedNativeExports(
|
||||
.all();
|
||||
expect(stored).toHaveLength(publication.overview.journalCoverage.transactionCount);
|
||||
expect(stored.length).toBeGreaterThan(accountRowCount);
|
||||
const activity = database
|
||||
.prepare(
|
||||
`SELECT activity.native_transaction_index, activity.context_json,
|
||||
transactions.native_index AS canonical_index
|
||||
FROM account_activity AS activity
|
||||
LEFT JOIN transactions ON transactions.native_index = activity.native_transaction_index`
|
||||
)
|
||||
.all();
|
||||
expect(activity.length).toBeGreaterThan(0);
|
||||
for (const record of activity) {
|
||||
expect(record.canonical_index).toBe(record.native_transaction_index);
|
||||
const context = JSON.parse(record.context_json as string);
|
||||
expect(Object.keys(context).sort()).toEqual([
|
||||
'accountPostingIndexes',
|
||||
'balance',
|
||||
'change',
|
||||
'investment',
|
||||
'isSplit',
|
||||
'otherAccounts'
|
||||
]);
|
||||
expect(JSON.stringify(context)).not.toMatch(
|
||||
/"(?:transaction|transactions|postings|tpostings)"\s*:/
|
||||
);
|
||||
}
|
||||
const pairs = stored.map((record) => {
|
||||
expect(record.original_json).toEqual(expect.any(String));
|
||||
expect(record.transacted_json).toEqual(expect.any(String));
|
||||
|
||||
@@ -4,6 +4,7 @@ import * as reports from './migrations/001_reports.js';
|
||||
import * as activityIndexes from './migrations/002_account_activity_indexes.js';
|
||||
import * as activitySummaries from './migrations/003_account_activity_summaries.js';
|
||||
import * as nativeTransactions from './migrations/004_native_transactions.js';
|
||||
import * as sharedAccountTransactions from './migrations/005_shared_account_transactions.js';
|
||||
import type { ReportDatabase } from './schema.js';
|
||||
|
||||
export async function runMigrations(database: Kysely<ReportDatabase>): Promise<void> {
|
||||
@@ -15,7 +16,8 @@ export async function runMigrations(database: Kysely<ReportDatabase>): Promise<v
|
||||
'001_reports': reports,
|
||||
'002_account_activity_indexes': activityIndexes,
|
||||
'003_account_activity_summaries': activitySummaries,
|
||||
'004_native_transactions': nativeTransactions
|
||||
'004_native_transactions': nativeTransactions,
|
||||
'005_shared_account_transactions': sharedAccountTransactions
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { sql, type Kysely } from 'kysely';
|
||||
import { up as createActivityIndexes } from './002_account_activity_indexes.js';
|
||||
|
||||
/** Fresh builders have no activity rows; published databases are never migrated in place. */
|
||||
export async function up(database: Kysely<unknown>): Promise<void> {
|
||||
await database.schema.dropTable('account_activity').execute();
|
||||
await database.schema
|
||||
.createTable('account_activity')
|
||||
.addColumn('account_name', 'text', (column) =>
|
||||
column.modifyFront(sql`COLLATE BINARY`).notNull()
|
||||
)
|
||||
.addColumn('ordinal', 'integer', (column) => column.notNull())
|
||||
.addColumn('account_date', 'text', (column) => column.notNull())
|
||||
.addColumn('native_transaction_index', 'integer', (column) => column.notNull())
|
||||
.addColumn('context_json', 'text', (column) => column.notNull())
|
||||
.addPrimaryKeyConstraint('account_activity_key', ['account_name', 'ordinal'])
|
||||
.addUniqueConstraint('account_activity_transaction', [
|
||||
'account_name',
|
||||
'native_transaction_index'
|
||||
])
|
||||
.addCheckConstraint('account_activity_ordinal', sql`ordinal >= 0`)
|
||||
.addCheckConstraint(
|
||||
'account_activity_context_json',
|
||||
sql`json_valid(context_json) AND json_type(context_json) = 'object'`
|
||||
)
|
||||
.addForeignKeyConstraint(
|
||||
'account_activity_native_transaction',
|
||||
['native_transaction_index'],
|
||||
'transactions',
|
||||
['native_index']
|
||||
)
|
||||
.addForeignKeyConstraint(
|
||||
'account_activity_account',
|
||||
['account_name'],
|
||||
'accounts',
|
||||
['name'],
|
||||
(constraint) => constraint.deferrable().initiallyDeferred()
|
||||
)
|
||||
.modifyEnd(sql`STRICT`)
|
||||
.execute();
|
||||
await createActivityIndexes(database);
|
||||
await sql`PRAGMA user_version = 3`.execute(database);
|
||||
}
|
||||
@@ -4,6 +4,12 @@ import type {
|
||||
PreparedAccountSummary,
|
||||
PreparedAccountRow
|
||||
} from '@hledger-reports/hledger';
|
||||
import {
|
||||
decodeNativeTransaction,
|
||||
prepareTransaction,
|
||||
type NativeTransactionPair
|
||||
} from '@hledger-reports/hledger/transactions';
|
||||
import { nativeTransactionFixture } from '../transactions/native.fixture.js';
|
||||
import { AccountActivitySummaryBuilder } from './account-activity-summary.js';
|
||||
|
||||
export function preparedActivitySummary(rows: readonly PreparedAccountRow[] = []) {
|
||||
@@ -67,13 +73,77 @@ export const amount = (quantity: string): DetailedAmount => ({
|
||||
style: null
|
||||
});
|
||||
export const sourcePosition = { file: '/private/books/included.journal', line: 12, column: 1 };
|
||||
export function accountTransactionExportsFixture(
|
||||
count = 1,
|
||||
description = 'exact producer strings',
|
||||
firstNativeIndex = 1
|
||||
): Array<{ originalJson: string; transactedJson: string }> {
|
||||
return Array.from({ length: count }, (_, i) => {
|
||||
const native = nativeTransactionFixture(firstNativeIndex + i, [
|
||||
'assets:cash',
|
||||
'expenses:食料',
|
||||
'expenses:household'
|
||||
]);
|
||||
const transaction = {
|
||||
...native,
|
||||
tdate: '2026-04-29',
|
||||
tdate2: '2026-04-30',
|
||||
tstatus: 'Cleared',
|
||||
tcode: 'REF-42',
|
||||
tdescription: `${description} ${i}`,
|
||||
tcomment: 'transaction note\nproject: home\n',
|
||||
tprecedingcomment: 'retained preceding comment\n',
|
||||
ttags: [
|
||||
['project', 'home'],
|
||||
['project', 'ordered']
|
||||
],
|
||||
tsourcepos: [12, 18].map((line) => ({
|
||||
sourceName: sourcePosition.file,
|
||||
sourceLine: line,
|
||||
sourceColumn: 1
|
||||
})),
|
||||
tpostings: native.tpostings.map((posting, index) => ({
|
||||
...posting,
|
||||
pamount: [
|
||||
{
|
||||
acommodity: '$',
|
||||
aquantity: {
|
||||
decimalMantissa: [-10000, 7500, 2500][index],
|
||||
decimalPlaces: 2,
|
||||
floatingPoint: [-100, 75, 25][index]
|
||||
},
|
||||
acost: null,
|
||||
acostbasis: null,
|
||||
astyle: null
|
||||
}
|
||||
],
|
||||
pdate: index === 0 ? '2026-04-30' : null,
|
||||
pdate2: null,
|
||||
pstatus: 'Unmarked',
|
||||
pcomment: ['bank note', 'food note', 'household note'][index],
|
||||
ptags: [['receipt', 'fictional']],
|
||||
pbalanceassertion: null
|
||||
}))
|
||||
};
|
||||
const json = JSON.stringify(transaction);
|
||||
return { originalJson: json, transactedJson: json };
|
||||
});
|
||||
}
|
||||
|
||||
export const preparedAccountRows = (
|
||||
count = 1,
|
||||
description = 'exact producer strings'
|
||||
description = 'exact producer strings',
|
||||
firstNativeIndex = 1
|
||||
): PreparedAccountRow[] =>
|
||||
Array.from(
|
||||
{ length: count },
|
||||
(_, i): PreparedAccountRow => ({
|
||||
preparedAccountRowsFromExports(
|
||||
accountTransactionExportsFixture(count, description, firstNativeIndex)
|
||||
);
|
||||
|
||||
export const preparedAccountRowsFromExports = (
|
||||
exports: readonly NativeTransactionPair[]
|
||||
): PreparedAccountRow[] =>
|
||||
exports.map(
|
||||
({ originalJson, transactedJson }): PreparedAccountRow => ({
|
||||
accountDate: '2026-04-30',
|
||||
isSplit: true,
|
||||
otherAccounts: ['expenses:食料', 'expenses:household'],
|
||||
@@ -95,53 +165,10 @@ export const preparedAccountRows = (
|
||||
gainPostingIndexes: [],
|
||||
issues: []
|
||||
},
|
||||
transaction: {
|
||||
index: i + 1,
|
||||
date: '2026-04-29',
|
||||
secondaryDate: '2026-04-30',
|
||||
status: 'Cleared',
|
||||
code: 'REF-42',
|
||||
description: `${description} ${i}`,
|
||||
payee: 'Market',
|
||||
note: 'supplies',
|
||||
comment: 'transaction note\nproject: home\n',
|
||||
precedingComment: 'retained preceding comment\n',
|
||||
tags: [
|
||||
['project', 'home'],
|
||||
['project', 'ordered']
|
||||
],
|
||||
postings: [
|
||||
['assets:cash', '-100.00', 'bank note'],
|
||||
['expenses:食料', '75.00', 'food note'],
|
||||
['expenses:household', '25.00', 'household note']
|
||||
].map(([account, quantity, comment], postingIndex) => {
|
||||
const date = postingIndex === 0 ? '2026-04-30' : null;
|
||||
const balanceAssertion =
|
||||
postingIndex === 0
|
||||
? {
|
||||
amount: amount('9007199254740993.123456789'),
|
||||
total: false,
|
||||
inclusive: false,
|
||||
sourcePosition
|
||||
}
|
||||
: null;
|
||||
return {
|
||||
account,
|
||||
amounts: [amount(quantity)],
|
||||
transactedAmounts: [amount(quantity)],
|
||||
date,
|
||||
secondaryDate: null,
|
||||
status: 'Unmarked',
|
||||
effectiveStatus: 'Cleared',
|
||||
comment,
|
||||
tags: [['receipt', 'fictional']],
|
||||
kind: 'RealPosting',
|
||||
balanceAssertion,
|
||||
transactionIndex: String(i + 1)
|
||||
};
|
||||
}),
|
||||
sourcePositions: [sourcePosition, { ...sourcePosition, line: 18 }]
|
||||
}
|
||||
transaction: prepareTransaction(
|
||||
decodeNativeTransaction(originalJson),
|
||||
decodeNativeTransaction(transactedJson)
|
||||
)
|
||||
})
|
||||
);
|
||||
export const ready = (name = 'assets:cash', rowCount = 1): PreparedAccountSummary => ({
|
||||
|
||||
@@ -95,29 +95,43 @@ export function hasOverviewContainers(value: unknown): value is OverviewData {
|
||||
);
|
||||
}
|
||||
|
||||
type AccountContext = Pick<
|
||||
PreparedAccountRow,
|
||||
'isSplit' | 'otherAccounts' | 'change' | 'balance' | 'accountPostingIndexes' | 'investment'
|
||||
>;
|
||||
|
||||
export function hasAccountContextContainers(value: unknown): value is AccountContext {
|
||||
return (
|
||||
isObject(value) &&
|
||||
typeof value.isSplit === 'boolean' &&
|
||||
Array.isArray(value.otherAccounts) &&
|
||||
value.otherAccounts.every((account) => typeof account === 'string') &&
|
||||
Array.isArray(value.change) &&
|
||||
Array.isArray(value.balance) &&
|
||||
Array.isArray(value.accountPostingIndexes) &&
|
||||
isObject(value.investment) &&
|
||||
Array.isArray(value.investment.securities) &&
|
||||
isObject(value.investment.cash) &&
|
||||
typeof value.investment.cash.direction === 'string' &&
|
||||
['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) &&
|
||||
Array.isArray(value.investment.cash.change) &&
|
||||
Array.isArray(value.investment.cash.balance) &&
|
||||
Array.isArray(value.investment.cash.unclassifiedChange) &&
|
||||
Array.isArray(value.investment.cash.unclassifiedBalance) &&
|
||||
Array.isArray(value.investment.cashPostingIndexes) &&
|
||||
Array.isArray(value.investment.incomePostingIndexes) &&
|
||||
Array.isArray(value.investment.expensePostingIndexes) &&
|
||||
Array.isArray(value.investment.gainPostingIndexes) &&
|
||||
Array.isArray(value.investment.issues)
|
||||
);
|
||||
}
|
||||
|
||||
export function hasAccountRowContainers(value: unknown): value is PreparedAccountRow {
|
||||
if (
|
||||
!isObject(value) ||
|
||||
typeof value.accountDate !== 'string' ||
|
||||
!Array.isArray(value.change) ||
|
||||
!Array.isArray(value.balance) ||
|
||||
!isObject(value.transaction) ||
|
||||
!Array.isArray(value.transaction.postings) ||
|
||||
!Array.isArray(value.accountPostingIndexes) ||
|
||||
!isObject(value.investment) ||
|
||||
!Array.isArray(value.investment.securities) ||
|
||||
!isObject(value.investment.cash) ||
|
||||
typeof value.investment.cash.direction !== 'string' ||
|
||||
!['in', 'out', 'mixed', 'none', 'unknown'].includes(value.investment.cash.direction) ||
|
||||
!Array.isArray(value.investment.cash.change) ||
|
||||
!Array.isArray(value.investment.cash.balance) ||
|
||||
!Array.isArray(value.investment.cash.unclassifiedChange) ||
|
||||
!Array.isArray(value.investment.cash.unclassifiedBalance) ||
|
||||
!Array.isArray(value.investment.cashPostingIndexes) ||
|
||||
!Array.isArray(value.investment.incomePostingIndexes) ||
|
||||
!Array.isArray(value.investment.expensePostingIndexes) ||
|
||||
!Array.isArray(value.investment.gainPostingIndexes) ||
|
||||
!Array.isArray(value.investment.issues) ||
|
||||
!value.transaction.postings.every(
|
||||
(posting) =>
|
||||
isObject(posting) &&
|
||||
@@ -126,8 +140,11 @@ export function hasAccountRowContainers(value: unknown): value is PreparedAccoun
|
||||
)
|
||||
return false;
|
||||
const postingCount = value.transaction.postings.length;
|
||||
return value.accountPostingIndexes.every(
|
||||
(index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount
|
||||
return (
|
||||
hasAccountContextContainers(value) &&
|
||||
value.accountPostingIndexes.every(
|
||||
(index) => Number.isSafeInteger(index) && index >= 0 && index < postingCount
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -17,14 +17,19 @@ import {
|
||||
writeAccountActivitySummaries,
|
||||
writeReportData
|
||||
} from './queries.js';
|
||||
import type { ReportTransaction } from '../database.js';
|
||||
import { insertNativeTransaction, attachTransactedTransaction } from '../transactions/index.js';
|
||||
import { richNativeExportsFixture } from '../transactions/native.fixture.js';
|
||||
import type { StoredReportData } from './data.js';
|
||||
import {
|
||||
amount,
|
||||
accountTransactionExportsFixture,
|
||||
ambiguous,
|
||||
preparedOverview,
|
||||
journal,
|
||||
ready,
|
||||
preparedAccountRows,
|
||||
preparedAccountRowsFromExports,
|
||||
preparedActivitySummary
|
||||
} from './data.fixture.js';
|
||||
|
||||
@@ -57,43 +62,49 @@ afterEach(async () => {
|
||||
}
|
||||
});
|
||||
|
||||
async function importTransactions(
|
||||
transaction: ReportTransaction,
|
||||
exports: Array<{ originalJson: string; transactedJson: string }>
|
||||
): Promise<void> {
|
||||
for (const { originalJson, transactedJson } of exports) {
|
||||
await insertNativeTransaction(transaction, originalJson);
|
||||
await attachTransactedTransaction(transaction, transactedJson);
|
||||
}
|
||||
}
|
||||
|
||||
function caseDistinctExports(firstNativeIndex: number) {
|
||||
return accountTransactionExportsFixture(1, 'case-distinct account', firstNativeIndex).map(
|
||||
({ originalJson }) => {
|
||||
const native = JSON.parse(originalJson);
|
||||
native.tpostings[0].paccount = 'assets:Cash';
|
||||
const json = JSON.stringify(native);
|
||||
return { originalJson: json, transactedJson: json };
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
it('stores exact prepared payloads and native row order using the migrated schema', async () => {
|
||||
const database = await createDatabase();
|
||||
const rows = preparedAccountRows(52).reverse();
|
||||
const rich = rows[0];
|
||||
// Preserve producer-owned prices and basis, including unavailable native conversions.
|
||||
rich.transaction.postings[1].amounts[0] = {
|
||||
...amount('75.00'),
|
||||
commodity: 'TEST',
|
||||
cost: { kind: 'UnitCost', amount: amount('0.000000000000000000012300') },
|
||||
costBasis: {
|
||||
amount: { ...amount('9007199254740993.123456789'), commodity: 'EUR' },
|
||||
date: '2026-04-01',
|
||||
label: 'retained basis label'
|
||||
},
|
||||
style: {
|
||||
commoditySide: 'L',
|
||||
commoditySpaced: false,
|
||||
decimalMark: '.',
|
||||
digitGroups: [',', [3]],
|
||||
precision: 2,
|
||||
rounding: 'NoRounding'
|
||||
}
|
||||
};
|
||||
rich.transaction.postings[1].transactedAmounts = [
|
||||
{ commodity: 'EUR', quantity: '0.000000000000000000922500' }
|
||||
const richExports = richNativeExportsFixture();
|
||||
richExports.transactedJson = richExports.transactedJson.replace(
|
||||
'"decimalMantissa": 10000',
|
||||
'"decimalMantissa": 9007199254740993123456789'
|
||||
);
|
||||
const exports = [
|
||||
richExports,
|
||||
...accountTransactionExportsFixture(51, 'exact producer strings', 8).reverse()
|
||||
];
|
||||
rich.transaction.postings[2].tags.push(['_conversion-posting', '']);
|
||||
rich.transaction.postings[2].transactedAmounts = null;
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
const rich = rows[0];
|
||||
rich.investment = {
|
||||
securities: [
|
||||
{
|
||||
commodity: 'TEST',
|
||||
commodity: 'USD',
|
||||
action: 'unknown',
|
||||
change: [{ commodity: 'TEST', quantity: '75.00' }],
|
||||
postingIndexes: [1],
|
||||
prices: [amount('0.000000000000000000012300')],
|
||||
transactedAmounts: rich.transaction.postings[1].transactedAmounts
|
||||
change: [{ commodity: 'USD', quantity: rich.transaction.postings[0].amounts[0].quantity }],
|
||||
postingIndexes: [0],
|
||||
prices: [rich.transaction.postings[0].amounts[0].cost!.amount],
|
||||
transactedAmounts: rich.transaction.postings[0].transactedAmounts
|
||||
}
|
||||
],
|
||||
cash: {
|
||||
@@ -117,8 +128,8 @@ it('stores exact prepared payloads and native row order using the migrated schem
|
||||
];
|
||||
nativeEnding.investment.cash.balance = [];
|
||||
nativeEnding.investment.cash.unclassifiedBalance = [{ commodity: 'EUR', quantity: '0.000' }];
|
||||
const otherRows = preparedAccountRows(1, 'case-distinct account');
|
||||
otherRows[0].transaction.postings[0].account = 'assets:Cash';
|
||||
const otherExports = caseDistinctExports(60);
|
||||
const otherRows = preparedAccountRowsFromExports(otherExports);
|
||||
const data: StoredReportData = {
|
||||
journal,
|
||||
overview: preparedOverview(),
|
||||
@@ -136,6 +147,7 @@ it('stores exact prepared payloads and native row order using the migrated schem
|
||||
]);
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, [...exports, ...otherExports]);
|
||||
await writeAccountRows(transaction, 'assets:cash', 0, rows.slice(0, 51));
|
||||
await writeAccountRows(transaction, 'assets:cash', 51, rows.slice(51));
|
||||
await writeAccountRows(transaction, 'assets:Cash', 0, otherRows);
|
||||
@@ -163,6 +175,9 @@ it('stores exact prepared payloads and native row order using the migrated schem
|
||||
offset: 50
|
||||
});
|
||||
expect(first).toEqual({ rows: rows.slice(0, 50), summary: summaries.get('assets:cash') });
|
||||
expect(first?.rows[0].transaction.postings[0].transactedAmounts).toEqual([
|
||||
{ commodity: 'USD', quantity: '90071992547409931234567.89' }
|
||||
]);
|
||||
expect(last).toEqual({ rows: rows.slice(50), summary: first?.summary });
|
||||
expect(await readAccountActivityPage(reader, data.accounts[2], firstPage)).toEqual({
|
||||
rows: otherRows,
|
||||
@@ -175,18 +190,18 @@ it('stores exact prepared payloads and native row order using the migrated schem
|
||||
expect((await stat(path)).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it('reads date-ordered pages with native ties and exact account scope', async () => {
|
||||
it('shares one canonical transaction without embedding its body in account context', async () => {
|
||||
const database = await createDatabase();
|
||||
const rows = preparedAccountRows(53).reverse();
|
||||
for (const [ordinal, row] of rows.entries()) {
|
||||
row.accountDate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
|
||||
row.transaction.postings[0].date = row.accountDate;
|
||||
}
|
||||
const otherRows = preparedAccountRows(1, 'case-distinct account');
|
||||
otherRows[0].transaction.postings[0].account = 'assets:Cash';
|
||||
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
|
||||
const other = { id: 'cash0002', ...ready('assets:Cash', otherRows.length) };
|
||||
await database.transaction().execute(async (transaction) => {
|
||||
const exports = accountTransactionExportsFixture();
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
const otherRows = preparedAccountRowsFromExports(exports);
|
||||
otherRows[0].accountPostingIndexes = [1];
|
||||
otherRows[0].otherAccounts = ['assets:cash', 'expenses:household'];
|
||||
const account = { id: 'cash0001', ...ready('assets:cash') };
|
||||
const other = { id: 'food0001', ...ready('expenses:食料') };
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, exports);
|
||||
await writeAccountRows(transaction, account.name, 0, rows);
|
||||
await writeAccountRows(transaction, other.name, 0, otherRows);
|
||||
await writeReportData(transaction, {
|
||||
@@ -201,7 +216,82 @@ it('reads date-ordered pages with native ties and exact account scope', async ()
|
||||
[other.name, preparedActivitySummary(otherRows)]
|
||||
])
|
||||
);
|
||||
});
|
||||
await transaction.commit().execute();
|
||||
} finally {
|
||||
if (!transaction.isCommitted && !transaction.isRolledBack)
|
||||
await transaction.rollback().execute();
|
||||
}
|
||||
expect(await database.selectFrom('transactions').select('native_index').execute()).toEqual([
|
||||
{ native_index: 1 }
|
||||
]);
|
||||
const stored = await database
|
||||
.selectFrom('account_activity')
|
||||
.selectAll()
|
||||
.orderBy('account_name')
|
||||
.execute();
|
||||
expect(stored.map((row) => row.native_transaction_index)).toEqual([1, 1]);
|
||||
for (const [ordinal, row] of stored.entries()) {
|
||||
const expected = ordinal === 0 ? rows[0] : otherRows[0];
|
||||
expect(JSON.parse(row.context_json)).toEqual({
|
||||
isSplit: expected.isSplit,
|
||||
otherAccounts: expected.otherAccounts,
|
||||
change: expected.change,
|
||||
balance: expected.balance,
|
||||
accountPostingIndexes: expected.accountPostingIndexes,
|
||||
investment: expected.investment
|
||||
});
|
||||
}
|
||||
const page = { order: 'asc', offset: 0, limit: 50 } as const;
|
||||
expect((await readAccountActivityPage(database, account, page))?.rows).toEqual(rows);
|
||||
expect((await readAccountActivityPage(database, other, page))?.rows).toEqual(otherRows);
|
||||
|
||||
// An account has one activity row per canonical transaction, not one per posting.
|
||||
await expect(writeAccountRows(database, account.name, 1, rows)).rejects.toThrow();
|
||||
await expect(
|
||||
writeAccountRows(database, account.name, 1, preparedAccountRows(1, 'not imported', 99))
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('reads date-ordered pages with native ties and exact account scope', async () => {
|
||||
const database = await createDatabase();
|
||||
const exports = accountTransactionExportsFixture(53)
|
||||
.reverse()
|
||||
.map(({ originalJson }, ordinal) => {
|
||||
const native = JSON.parse(originalJson);
|
||||
native.tpostings[0].pdate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
|
||||
const json = JSON.stringify(native);
|
||||
return { originalJson: json, transactedJson: json };
|
||||
});
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
for (const [ordinal, row] of rows.entries()) {
|
||||
row.accountDate = ordinal < 26 ? '2026-04-30' : '2026-04-29';
|
||||
}
|
||||
const otherExports = caseDistinctExports(54);
|
||||
const otherRows = preparedAccountRowsFromExports(otherExports);
|
||||
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
|
||||
const other = { id: 'cash0002', ...ready('assets:Cash', otherRows.length) };
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, [...exports, ...otherExports]);
|
||||
await writeAccountRows(transaction, account.name, 0, rows);
|
||||
await writeAccountRows(transaction, other.name, 0, otherRows);
|
||||
await writeReportData(transaction, {
|
||||
journal,
|
||||
overview: preparedOverview(),
|
||||
accounts: [account, other]
|
||||
});
|
||||
await writeAccountActivitySummaries(
|
||||
transaction,
|
||||
new Map([
|
||||
[account.name, preparedActivitySummary(rows)],
|
||||
[other.name, preparedActivitySummary(otherRows)]
|
||||
])
|
||||
);
|
||||
await transaction.commit().execute();
|
||||
} finally {
|
||||
if (!transaction.isCommitted && !transaction.isRolledBack)
|
||||
await transaction.rollback().execute();
|
||||
}
|
||||
const resultRowCounts: number[] = [];
|
||||
const pageReader = database.withPlugin({
|
||||
transformQuery: ({ node }) => node,
|
||||
@@ -245,22 +335,39 @@ it('reads date-ordered pages with native ties and exact account scope', async ()
|
||||
it('decodes only the requested page and rejects invalid posting references', async () => {
|
||||
const database = await createDatabase();
|
||||
const account = { id: 'cash0001', ...ready('assets:cash', 2) };
|
||||
const rows = preparedAccountRows(2);
|
||||
const exports = accountTransactionExportsFixture(2);
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
await writeReportData(database, { journal, overview: preparedOverview(), accounts: [account] });
|
||||
await writeAccountActivitySummaries(
|
||||
database,
|
||||
new Map([[account.name, preparedActivitySummary(rows)]])
|
||||
);
|
||||
await writeAccountRows(database, account.name, 0, rows);
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, exports);
|
||||
await writeAccountRows(transaction, account.name, 0, rows);
|
||||
await transaction.commit().execute();
|
||||
} finally {
|
||||
if (!transaction.isCommitted && !transaction.isRolledBack)
|
||||
await transaction.rollback().execute();
|
||||
}
|
||||
await closeDatabase(database);
|
||||
|
||||
// Valid SQLite/JSON is not sufficient: saved posting references must still be usable.
|
||||
const native = new DatabaseSync(path);
|
||||
try {
|
||||
const badRow = { ...rows[1], accountPostingIndexes: [rows[1].transaction.postings.length] };
|
||||
const saved = native
|
||||
.prepare('SELECT context_json FROM account_activity WHERE ordinal = 1')
|
||||
.get();
|
||||
const badContext = {
|
||||
...JSON.parse(String(saved!.context_json)),
|
||||
accountPostingIndexes: [rows[1].transaction.postings.length]
|
||||
};
|
||||
native
|
||||
.prepare('UPDATE account_activity SET row_json = ? WHERE account_name = ? AND ordinal = 1')
|
||||
.run(JSON.stringify(badRow), account.name);
|
||||
.prepare(
|
||||
'UPDATE account_activity SET context_json = ? WHERE account_name = ? AND ordinal = 1'
|
||||
)
|
||||
.run(JSON.stringify(badContext), account.name);
|
||||
} finally {
|
||||
native.close();
|
||||
}
|
||||
@@ -276,12 +383,74 @@ it('decodes only the requested page and rejects invalid posting references', asy
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('rejects selected missing or invalid canonical exports without decoding off-page transactions', async () => {
|
||||
const database = await createDatabase();
|
||||
const exports = accountTransactionExportsFixture(4);
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
const account = { id: 'cash0001', ...ready('assets:cash', rows.length) };
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, exports);
|
||||
await writeAccountRows(transaction, account.name, 0, rows);
|
||||
await writeReportData(transaction, {
|
||||
journal,
|
||||
overview: preparedOverview(),
|
||||
accounts: [account]
|
||||
});
|
||||
await writeAccountActivitySummaries(
|
||||
transaction,
|
||||
new Map([[account.name, preparedActivitySummary(rows)]])
|
||||
);
|
||||
await transaction.commit().execute();
|
||||
} finally {
|
||||
if (!transaction.isCommitted && !transaction.isRolledBack)
|
||||
await transaction.rollback().execute();
|
||||
}
|
||||
await closeDatabase(database);
|
||||
|
||||
const native = new DatabaseSync(path);
|
||||
try {
|
||||
// Simulate damaged publication storage, including an orphan that normal writes forbid.
|
||||
native.exec('PRAGMA foreign_keys = OFF');
|
||||
const wrongIdentity = accountTransactionExportsFixture(1, 'wrong canonical identity', 99)[0];
|
||||
native
|
||||
.prepare(
|
||||
'UPDATE transactions SET original_json = ?, transacted_json = ? WHERE native_index = 2'
|
||||
)
|
||||
.run(wrongIdentity.originalJson, wrongIdentity.transactedJson);
|
||||
native.prepare('UPDATE transactions SET transacted_json = NULL WHERE native_index = 3').run();
|
||||
native.prepare('DELETE FROM transactions WHERE native_index = 1').run();
|
||||
} finally {
|
||||
native.close();
|
||||
}
|
||||
const reader = await openReadOnlyReportDatabase(path);
|
||||
if (!reader) throw new Error('Expected the current report schema.');
|
||||
databases.add(reader);
|
||||
expect((await readReportData(reader)).accounts).toEqual([account]);
|
||||
const readRow = (offset: number) =>
|
||||
readAccountActivityPage(reader, account, { order: 'asc', offset, limit: 1 });
|
||||
// The orphan still occupies its ordinal; a join must not shift later rows into the page.
|
||||
expect(await readRow(3)).toEqual({ rows: rows.slice(3), summary: preparedActivitySummary(rows) });
|
||||
await expect(readRow(0)).rejects.toThrow();
|
||||
await expect(readRow(1)).rejects.toThrow();
|
||||
await expect(readRow(2)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('rejects missing summaries and malformed native bags rather than inferring from a page', async () => {
|
||||
const database = await createDatabase();
|
||||
const account = { id: 'cash0001', ...ready() };
|
||||
const rows = preparedAccountRows();
|
||||
const exports = accountTransactionExportsFixture();
|
||||
const rows = preparedAccountRowsFromExports(exports);
|
||||
await writeReportData(database, { journal, overview: preparedOverview(), accounts: [account] });
|
||||
await writeAccountRows(database, account.name, 0, rows);
|
||||
const transaction = await database.startTransaction().execute();
|
||||
try {
|
||||
await importTransactions(transaction, exports);
|
||||
await writeAccountRows(transaction, account.name, 0, rows);
|
||||
await transaction.commit().execute();
|
||||
} finally {
|
||||
if (!transaction.isCommitted && !transaction.isRolledBack)
|
||||
await transaction.rollback().execute();
|
||||
}
|
||||
const page = { order: 'asc', offset: 0, limit: 50 } as const;
|
||||
await expect(readAccountActivityPage(database, account, page)).rejects.toThrow();
|
||||
await expect(writeAccountActivitySummaries(database, new Map())).rejects.toThrow();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { PreparedAccountRow } from '@hledger-reports/hledger';
|
||||
import { decodeNativeTransaction, prepareTransaction } from '@hledger-reports/hledger/transactions';
|
||||
import type { Kysely } from 'kysely';
|
||||
import type { ReportDatabase } from '../schema.js';
|
||||
import {
|
||||
@@ -7,6 +8,7 @@ import {
|
||||
type AccountActivitySummary
|
||||
} from './account-activity-summary.js';
|
||||
import {
|
||||
hasAccountContextContainers,
|
||||
hasAccountRowContainers,
|
||||
hasStoredReportData,
|
||||
type StoredAccount,
|
||||
@@ -29,8 +31,12 @@ export interface AccountActivityPageOptions {
|
||||
|
||||
type StoredActivityRow = Pick<
|
||||
ReportDatabase['account_activity'],
|
||||
'ordinal' | 'account_date' | 'row_json'
|
||||
>;
|
||||
'ordinal' | 'account_date' | 'native_transaction_index' | 'context_json'
|
||||
> & {
|
||||
canonical_native_index: number | null;
|
||||
original_json: string | null;
|
||||
transacted_json: string | null;
|
||||
};
|
||||
|
||||
export async function writeAccountRows(
|
||||
database: Kysely<ReportDatabase>,
|
||||
@@ -42,12 +48,25 @@ export async function writeAccountRows(
|
||||
throw new Error('Invalid account rows.');
|
||||
for (let start = 0; start < rows.length; start += insertBatchSize) {
|
||||
const batch = rows.slice(start, start + insertBatchSize).map((row, index) => {
|
||||
if (!hasAccountRowContainers(row)) throw new Error('Invalid account rows.');
|
||||
if (
|
||||
!hasAccountRowContainers(row) ||
|
||||
!Number.isSafeInteger(row.transaction.index) ||
|
||||
row.transaction.index <= 0
|
||||
)
|
||||
throw new Error('Invalid account rows.');
|
||||
return {
|
||||
account_name: name,
|
||||
ordinal: firstOrdinal + start + index,
|
||||
account_date: row.accountDate,
|
||||
row_json: JSON.stringify(row)
|
||||
native_transaction_index: row.transaction.index,
|
||||
context_json: JSON.stringify({
|
||||
isSplit: row.isSplit,
|
||||
otherAccounts: row.otherAccounts,
|
||||
change: row.change,
|
||||
balance: row.balance,
|
||||
accountPostingIndexes: row.accountPostingIndexes,
|
||||
investment: row.investment
|
||||
})
|
||||
};
|
||||
});
|
||||
await database.insertInto('account_activity').values(batch).execute();
|
||||
@@ -143,15 +162,37 @@ export async function readReportData(database: Kysely<ReportDatabase>): Promise<
|
||||
}
|
||||
|
||||
function decodeAccountRow(record: StoredActivityRow, rowCount: number): PreparedAccountRow {
|
||||
const row = decodePayload(record.row_json);
|
||||
if (
|
||||
!Number.isSafeInteger(record.ordinal) ||
|
||||
record.ordinal < 0 ||
|
||||
record.ordinal >= rowCount ||
|
||||
!hasAccountRowContainers(row) ||
|
||||
record.account_date !== row.accountDate
|
||||
!Number.isSafeInteger(record.native_transaction_index) ||
|
||||
record.native_transaction_index <= 0 ||
|
||||
record.canonical_native_index !== record.native_transaction_index ||
|
||||
typeof record.original_json !== 'string' ||
|
||||
typeof record.transacted_json !== 'string'
|
||||
)
|
||||
throw new Error('Invalid account data.');
|
||||
const context = decodePayload(record.context_json);
|
||||
if (!hasAccountContextContainers(context)) throw new Error('Invalid account data.');
|
||||
const original = decodeNativeTransaction(record.original_json);
|
||||
const transacted = decodeNativeTransaction(record.transacted_json);
|
||||
if (
|
||||
original.tindex !== record.native_transaction_index ||
|
||||
transacted.tindex !== record.native_transaction_index
|
||||
)
|
||||
throw new Error('Invalid account data.');
|
||||
const row = {
|
||||
accountDate: record.account_date,
|
||||
isSplit: context.isSplit,
|
||||
otherAccounts: context.otherAccounts,
|
||||
change: context.change,
|
||||
balance: context.balance,
|
||||
accountPostingIndexes: context.accountPostingIndexes,
|
||||
investment: context.investment,
|
||||
transaction: prepareTransaction(original, transacted)
|
||||
};
|
||||
if (!hasAccountRowContainers(row)) throw new Error('Invalid account data.');
|
||||
return row;
|
||||
}
|
||||
|
||||
@@ -233,10 +274,23 @@ export async function readAccountActivityPage(
|
||||
const summary = decodeActivitySummary(storedSummary);
|
||||
const records = await database
|
||||
.selectFrom('account_activity')
|
||||
.select(['ordinal', 'account_date', 'row_json'])
|
||||
.where('account_name', '=', account.name)
|
||||
.orderBy('account_date', order)
|
||||
.orderBy('ordinal', 'asc')
|
||||
.leftJoin(
|
||||
'transactions',
|
||||
'transactions.native_index',
|
||||
'account_activity.native_transaction_index'
|
||||
)
|
||||
.select([
|
||||
'account_activity.ordinal',
|
||||
'account_activity.account_date',
|
||||
'account_activity.native_transaction_index',
|
||||
'account_activity.context_json',
|
||||
'transactions.native_index as canonical_native_index',
|
||||
'transactions.original_json',
|
||||
'transactions.transacted_json'
|
||||
])
|
||||
.where('account_activity.account_name', '=', account.name)
|
||||
.orderBy('account_activity.account_date', order)
|
||||
.orderBy('account_activity.ordinal', 'asc')
|
||||
.limit(limit)
|
||||
.offset(offset)
|
||||
.execute();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const reportSchemaVersion = 2;
|
||||
export const reportSchemaVersion = 3;
|
||||
|
||||
interface PublicationTable {
|
||||
singleton: 1;
|
||||
@@ -20,7 +20,8 @@ interface AccountActivityTable {
|
||||
account_name: string;
|
||||
ordinal: number;
|
||||
account_date: string;
|
||||
row_json: string;
|
||||
native_transaction_index: number;
|
||||
context_json: string;
|
||||
}
|
||||
|
||||
interface AccountActivitySummaryTable {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export { decodeNativeTransaction } from './export.js';
|
||||
export { assertTransactionExportsMatch } from './transaction.js';
|
||||
export { assertTransactionExportsMatch, prepareTransaction } from './transaction.js';
|
||||
export type { NativeTransaction } from './transaction.js';
|
||||
export type { NativeTransactionPair, PreparationTransactions } from './store.js';
|
||||
@@ -13,9 +13,10 @@ import {
|
||||
readOverview,
|
||||
readReportPublication
|
||||
} from '../index.js';
|
||||
import type { ReportPublication, StoredAccount } from '../index.js';
|
||||
import type { ReportPublication, ReportStage, StoredAccount } from '../index.js';
|
||||
import {
|
||||
ambiguous,
|
||||
accountTransactionExportsFixture,
|
||||
preparedOverview as fixture,
|
||||
journal,
|
||||
ready,
|
||||
@@ -46,12 +47,25 @@ const newer = { id: 'newer', createdAt: new Date('2026-05-01T00:00:00.000Z') };
|
||||
const safeError = (message: string) => (error: Error) =>
|
||||
error.message === message && error.cause === undefined;
|
||||
|
||||
async function importNativeExports(
|
||||
stage: ReportStage,
|
||||
count = 1,
|
||||
description = 'exact producer strings',
|
||||
firstIndex = 1
|
||||
) {
|
||||
for (const pair of accountTransactionExportsFixture(count, description, firstIndex)) {
|
||||
await stage.transactions.insertOriginal(pair.originalJson);
|
||||
await stage.transactions.attachTransacted(pair.transactedJson);
|
||||
}
|
||||
}
|
||||
|
||||
async function publish(
|
||||
run = older,
|
||||
data = fixture(),
|
||||
previousAccounts?: readonly StoredAccount[] | null
|
||||
) {
|
||||
const stage = await createReportStage(snapshots, run, previousAccounts);
|
||||
await importNativeExports(stage);
|
||||
await stage.writeRows('assets:cash', rows());
|
||||
return stage.publish(data, [ready()], journal);
|
||||
}
|
||||
@@ -94,6 +108,7 @@ it('commits and closes a complete private database before making its directory v
|
||||
return database;
|
||||
});
|
||||
const stage = await createReportStage(snapshots, newer, pinned.accounts);
|
||||
await importNativeExports(stage, 52);
|
||||
const transactions = rows(52);
|
||||
transactions[0].change.push({
|
||||
...transactions[0].change[0],
|
||||
@@ -122,10 +137,22 @@ it('commits and closes a complete private database before making its directory v
|
||||
expect(JSON.parse(record.overview_json as string)).toEqual(fixture());
|
||||
expect(
|
||||
database
|
||||
.prepare('SELECT row_json FROM account_activity ORDER BY ordinal')
|
||||
.prepare(
|
||||
'SELECT account_date, native_transaction_index, context_json FROM account_activity ORDER BY ordinal'
|
||||
)
|
||||
.all()
|
||||
.map((row) => JSON.parse(row.row_json as string))
|
||||
).toEqual(transactions);
|
||||
.map((row) => ({
|
||||
accountDate: row.account_date,
|
||||
nativeIndex: row.native_transaction_index,
|
||||
context: JSON.parse(row.context_json as string)
|
||||
}))
|
||||
).toEqual(
|
||||
transactions.map(({ accountDate, transaction, ...context }) => ({
|
||||
accountDate,
|
||||
nativeIndex: transaction.index,
|
||||
context
|
||||
}))
|
||||
);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
@@ -297,6 +324,7 @@ it('reuses IDs by exact account identity and avoids collisions with removed and
|
||||
generated.push(0, 1, 2, 2, 3);
|
||||
const stage = await createReportStage(snapshots, newer, previous.accounts);
|
||||
const names = ['資産/../銀行:Ω', 'new:Ω', 'NEW:Ω'];
|
||||
await importNativeExports(stage);
|
||||
for (const name of names) await stage.writeRows(name, rows());
|
||||
await stage.publish(
|
||||
fixture(),
|
||||
@@ -323,6 +351,7 @@ it('keeps completed generations pinned through same-request recovery and discard
|
||||
await publish();
|
||||
const pinned = await selected();
|
||||
const stage = await createReportStage(snapshots, older, pinned.accounts);
|
||||
await importNativeExports(stage, 1, 'recovered');
|
||||
await stage.writeRows('assets:cash', rows(1, 'recovered'));
|
||||
expect(await selected()).toEqual(pinned);
|
||||
await stage.publish({ ...fixture(), preparedAt: 'recovered' }, [ready()], journal);
|
||||
@@ -517,6 +546,7 @@ it.each([
|
||||
await publish();
|
||||
const pinned = await selected();
|
||||
const stage = await createReportStage(snapshots, newer, pinned.accounts);
|
||||
await importNativeExports(stage);
|
||||
await stage.writeRows('assets:cash', rows());
|
||||
await expect(stage.publish(fixture(), [summary], journal)).rejects.toSatisfy(
|
||||
safeError('Unable to prepare reports')
|
||||
@@ -531,6 +561,7 @@ it('cleans only its stage when the publication directory rename fails', async ()
|
||||
const sets = join(snapshots, '.sets');
|
||||
await fs.mkdir(join(sets, '.foreign.tmp'));
|
||||
const stage = await createReportStage(snapshots, newer, pinned.accounts);
|
||||
await importNativeExports(stage);
|
||||
await stage.writeRows('assets:cash', rows());
|
||||
vi.spyOn(fs, 'rename').mockRejectedValue(new Error(`${snapshots}: private rename failure`));
|
||||
await expect(stage.publish(fixture(), [ready()], journal)).rejects.toSatisfy(
|
||||
@@ -556,6 +587,7 @@ it('rejects noncontiguous writes and retries removal without closing the databas
|
||||
return database;
|
||||
});
|
||||
const stage = await createReportStage(snapshots, older);
|
||||
await importNativeExports(stage);
|
||||
await stage.writeRows('a', rows());
|
||||
await stage.writeRows('b', rows());
|
||||
const remove = fs.rm;
|
||||
@@ -592,6 +624,7 @@ it('publishes without financial readback and keeps the new generation after alia
|
||||
.spyOn(databases, 'openReadOnlyReportDatabase')
|
||||
.mockRejectedValue(new Error('Publication must not read selected financial data'));
|
||||
const stage = await createReportStage(snapshots, newer, pinned.accounts);
|
||||
await importNativeExports(stage, 1, 'committed');
|
||||
await stage.writeRows('assets:cash', rows(1, 'committed'));
|
||||
const rename = fs.rename;
|
||||
vi.spyOn(fs, 'rename').mockImplementation(async (source, destination) => {
|
||||
@@ -702,14 +735,14 @@ it('reports a bad selected database without falling back or changing its bytes',
|
||||
expect(previousPage?.rows).toEqual(rows());
|
||||
});
|
||||
|
||||
it('withholds a selected v1 SQLite publication until explicit preparation without changing its bytes', async () => {
|
||||
it('withholds a selected v2 SQLite publication until explicit preparation without changing its bytes', async () => {
|
||||
await publish();
|
||||
const previous = await selected();
|
||||
await publish(newer);
|
||||
const path = join((await selected()).directory, 'reports.sqlite');
|
||||
const database = new DatabaseSync(path);
|
||||
try {
|
||||
database.exec('PRAGMA user_version = 1');
|
||||
database.exec('PRAGMA user_version = 2');
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
@@ -728,25 +761,21 @@ it('withholds a selected v1 SQLite publication until explicit preparation withou
|
||||
expect(await fs.readFile(path)).toEqual(bytes);
|
||||
});
|
||||
|
||||
it('leaves a selected JSON-only generation unavailable and untouched until explicit preparation', async () => {
|
||||
it('requires selected SQLite even when an old financial JSON file exists', async () => {
|
||||
await publish();
|
||||
const previous = await selected();
|
||||
const token = `${String(newer.createdAt.getTime()).padStart(16, '0')}-${newer.id}-00000000-0000-4000-8000-000000000000`;
|
||||
const directory = join(snapshots, '.sets', token);
|
||||
await fs.mkdir(join(directory, 'transactions'), { recursive: true, mode: 0o700 });
|
||||
const overviewPath = join(directory, 'overview.json');
|
||||
const accountPath = join(directory, 'transactions', `${previous.accounts[0].id}.json`);
|
||||
const overviewBytes = JSON.stringify({
|
||||
format: 8,
|
||||
journal,
|
||||
overview: fixture(),
|
||||
accounts: previous.accounts
|
||||
});
|
||||
const accountBytes = 'opaque legacy account payload';
|
||||
await fs.writeFile(overviewPath, overviewBytes);
|
||||
await fs.writeFile(accountPath, accountBytes);
|
||||
expect(await readReportPublication(snapshots)).toBeNull();
|
||||
expect(await readOverview(snapshots)).toBeNull();
|
||||
await fs.mkdir(directory, { mode: 0o700 });
|
||||
const jsonPath = join(directory, 'overview.json');
|
||||
const bytes = Buffer.from('{"obsolete":"fictional private report"}');
|
||||
await fs.writeFile(jsonPath, bytes);
|
||||
await expect(readReportPublication(snapshots)).rejects.toSatisfy(
|
||||
safeError('Unable to read overview')
|
||||
);
|
||||
await expect(readOverview(snapshots)).rejects.toSatisfy(safeError('Unable to read overview'));
|
||||
expect(await fs.readFile(jsonPath)).toEqual(bytes);
|
||||
expect(await fs.readdir(directory)).toEqual(['overview.json']);
|
||||
const previousPage = await readAccountPage(previous, previous.accounts[0].id, {
|
||||
order: 'asc',
|
||||
offset: 0,
|
||||
@@ -755,9 +784,8 @@ it('leaves a selected JSON-only generation unavailable and untouched until expli
|
||||
expect(previousPage?.rows).toEqual(rows());
|
||||
await publish({ id: 'explicit', createdAt: new Date(newer.createdAt.getTime() + 1) });
|
||||
expect(await readOverview(snapshots)).toEqual(fixture());
|
||||
expect(await fs.readFile(overviewPath, 'utf8')).toBe(overviewBytes);
|
||||
expect(await fs.readFile(accountPath, 'utf8')).toBe(accountBytes);
|
||||
expect((await fs.readdir(directory)).sort()).toEqual(['overview.json', 'transactions']);
|
||||
expect(await fs.readFile(jsonPath)).toEqual(bytes);
|
||||
expect(await fs.readdir(directory)).toEqual(['overview.json']);
|
||||
});
|
||||
|
||||
it('rejects a symlink in place of the selected completed directory', async () => {
|
||||
|
||||
@@ -40,7 +40,7 @@ import {
|
||||
type StoredNativeTransaction
|
||||
} from '@hledger-reports/db/transactions';
|
||||
import { randomInt, randomUUID } from 'node:crypto';
|
||||
import { lstat, mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
|
||||
import { mkdir, rename, rm, symlink, unlink } from 'node:fs/promises';
|
||||
import { basename, dirname, join } from 'node:path';
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import { namesIn, privateDirectory } from '../files.js';
|
||||
@@ -88,17 +88,6 @@ async function latestSet(snapshotDirectory: string): Promise<string | undefined>
|
||||
.at(-1);
|
||||
}
|
||||
|
||||
async function openSelectedDatabase(directory: string): Promise<ReportConnection | null> {
|
||||
try {
|
||||
return await openReadOnlyReportDatabase(join(directory, databaseName));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
||||
const legacy = await lstat(join(directory, 'overview.json'));
|
||||
if (!legacy.isFile()) throw error;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readReportPublication(
|
||||
snapshotDirectory: string
|
||||
): Promise<ReportPublication | null> {
|
||||
@@ -107,7 +96,7 @@ export async function readReportPublication(
|
||||
if (!latest) return null;
|
||||
const directory = join(snapshotDirectory, '.sets', latest);
|
||||
await privateDirectory(directory);
|
||||
const database = await openSelectedDatabase(directory);
|
||||
const database = await openReadOnlyReportDatabase(join(directory, databaseName));
|
||||
if (!database) return null;
|
||||
try {
|
||||
const data = await readReportData(database);
|
||||
|
||||
Reference in new issue
Block a user