Native transaction storage and metadata foundation #49
No files matched your search
@@ -13,6 +13,11 @@
|
||||
"development": "./src/reports/index.ts",
|
||||
"types": "./dist/src/reports/index.d.ts",
|
||||
"import": "./dist/src/reports/index.js"
|
||||
},
|
||||
"./transactions": {
|
||||
"development": "./src/transactions/index.ts",
|
||||
"types": "./dist/src/transactions/index.d.ts",
|
||||
"import": "./dist/src/transactions/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
|
||||
@@ -3,6 +3,7 @@ import { Migrator } from 'kysely/migration';
|
||||
import * as reports from './migrations/001_reports.js';
|
||||
import * as activityIndexes from './migrations/002_account_activity_indexes.js';
|
||||
import * as activitySummaries from './migrations/003_account_activity_summaries.js';
|
||||
import * as nativeTransactions from './migrations/004_native_transactions.js';
|
||||
import type { ReportDatabase } from './schema.js';
|
||||
|
||||
export async function runMigrations(database: Kysely<ReportDatabase>): Promise<void> {
|
||||
@@ -13,7 +14,8 @@ export async function runMigrations(database: Kysely<ReportDatabase>): Promise<v
|
||||
return {
|
||||
'001_reports': reports,
|
||||
'002_account_activity_indexes': activityIndexes,
|
||||
'003_account_activity_summaries': activitySummaries
|
||||
'003_account_activity_summaries': activitySummaries,
|
||||
'004_native_transactions': nativeTransactions
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import { sql, type Kysely } from 'kysely';
|
||||
|
||||
export async function up(database: Kysely<unknown>): Promise<void> {
|
||||
await database.schema
|
||||
.createTable('transactions')
|
||||
.addColumn('native_index', 'integer', (column) => column.notNull().primaryKey())
|
||||
.addColumn('transaction_date', 'text', (column) => column.notNull())
|
||||
.addColumn('status', 'text', (column) => column.notNull())
|
||||
.addColumn('original_json', 'text', (column) => column.notNull())
|
||||
.addColumn('transacted_json', 'text')
|
||||
.addCheckConstraint(
|
||||
'transactions_native_index',
|
||||
sql`native_index > 0 AND native_index <= 9007199254740991`
|
||||
)
|
||||
.addCheckConstraint(
|
||||
'transactions_original_json',
|
||||
sql`json_valid(original_json) AND json_type(original_json) = 'object'`
|
||||
)
|
||||
.addCheckConstraint(
|
||||
'transactions_transacted_json',
|
||||
sql`transacted_json IS NULL
|
||||
OR (json_valid(transacted_json) AND json_type(transacted_json) = 'object')`
|
||||
)
|
||||
.modifyEnd(sql`STRICT`)
|
||||
.execute();
|
||||
|
||||
await database.schema
|
||||
.createTable('transaction_postings')
|
||||
.addColumn('native_transaction_index', 'integer', (column) => column.notNull())
|
||||
.addColumn('posting_ordinal', 'integer', (column) => column.notNull())
|
||||
.addColumn('account_name', 'text', (column) =>
|
||||
column.modifyFront(sql`COLLATE BINARY`).notNull()
|
||||
)
|
||||
.addColumn('posting_date', 'text')
|
||||
.addColumn('status', 'text', (column) => column.notNull())
|
||||
.addPrimaryKeyConstraint('transaction_postings_key', [
|
||||
'native_transaction_index',
|
||||
'posting_ordinal'
|
||||
])
|
||||
.addCheckConstraint('transaction_postings_ordinal', sql`posting_ordinal >= 0`)
|
||||
.addForeignKeyConstraint(
|
||||
'transaction_postings_transaction',
|
||||
['native_transaction_index'],
|
||||
'transactions',
|
||||
['native_index']
|
||||
)
|
||||
.modifyEnd(sql`STRICT`)
|
||||
.execute();
|
||||
|
||||
await database.schema
|
||||
.createIndex('transaction_postings_account')
|
||||
.on('transaction_postings')
|
||||
.columns(['account_name', 'native_transaction_index', 'posting_ordinal'])
|
||||
.execute();
|
||||
}
|
||||
@@ -32,9 +32,27 @@ interface AccountActivitySummaryTable {
|
||||
unclassified_ending_cash_json: string;
|
||||
}
|
||||
|
||||
interface NativeTransactionTable {
|
||||
native_index: number;
|
||||
transaction_date: string;
|
||||
status: string;
|
||||
original_json: string;
|
||||
transacted_json: string | null;
|
||||
}
|
||||
|
||||
interface TransactionPostingTable {
|
||||
native_transaction_index: number;
|
||||
posting_ordinal: number;
|
||||
account_name: string;
|
||||
posting_date: string | null;
|
||||
status: string;
|
||||
}
|
||||
|
||||
export interface ReportDatabase {
|
||||
publication: PublicationTable;
|
||||
accounts: AccountTable;
|
||||
account_activity: AccountActivityTable;
|
||||
account_activity_summaries: AccountActivitySummaryTable;
|
||||
transactions: NativeTransactionTable;
|
||||
transaction_postings: TransactionPostingTable;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export {
|
||||
insertNativeTransaction,
|
||||
attachTransactedTransaction,
|
||||
readNativeTransaction,
|
||||
readAccountTransactionBatch,
|
||||
assertTransactionImportComplete
|
||||
} from './queries.js';
|
||||
export type { StoredNativeTransaction } from './queries.js';
|
||||
@@ -0,0 +1,80 @@
|
||||
import { transactionFixture } from '../../../hledger/src/transactions/transaction.fixture.js';
|
||||
|
||||
export function nativeTransactionFixture(
|
||||
nativeIndex = 7,
|
||||
accounts = ['assets:cash', 'expenses:office', 'assets:cash']
|
||||
) {
|
||||
const native = transactionFixture();
|
||||
return {
|
||||
...native,
|
||||
tindex: nativeIndex,
|
||||
tcode: `REF-${nativeIndex}`,
|
||||
tpostings: accounts.map((account, ordinal) => ({
|
||||
...native.tpostings[ordinal % native.tpostings.length],
|
||||
paccount: account,
|
||||
ptransaction_: String(nativeIndex)
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
export function richNativeExportsFixture() {
|
||||
const native = nativeTransactionFixture(7, [
|
||||
'assets:cash',
|
||||
'expenses:office',
|
||||
'assets:cash',
|
||||
'equity:conversion'
|
||||
]);
|
||||
const leaf = native.tpostings[1].pamount[0];
|
||||
const original = {
|
||||
...native,
|
||||
futureNativeField: { retained: ['unknown', 'ordered'] },
|
||||
tpostings: native.tpostings.map((posting, ordinal) => {
|
||||
if (ordinal === 3) {
|
||||
return { ...posting, ptags: [...posting.ptags, ['_conversion-posting', '']] };
|
||||
}
|
||||
if (ordinal !== 0) return posting;
|
||||
return {
|
||||
...posting,
|
||||
futurePostingField: 'retain me',
|
||||
pamount: [
|
||||
{
|
||||
...leaf,
|
||||
aquantity: {
|
||||
decimalMantissa: '__EXACT_NATIVE_MANTISSA__',
|
||||
decimalPlaces: 4,
|
||||
floatingPoint: 900
|
||||
},
|
||||
acost: { tag: 'UnitCost', contents: leaf },
|
||||
acostbasis: { cbCost: leaf, cbDate: '2026-04-01', cbLabel: 'fictional lot' },
|
||||
astyle: {
|
||||
...leaf.astyle,
|
||||
asdigitgroups: [',', [3, 2]],
|
||||
asrounding: 'NoRounding'
|
||||
}
|
||||
}
|
||||
],
|
||||
pbalanceassertion: {
|
||||
baamount: leaf,
|
||||
batotal: true,
|
||||
bainclusive: false,
|
||||
baposition: native.tsourcepos[0]
|
||||
}
|
||||
};
|
||||
})
|
||||
};
|
||||
const transacted = {
|
||||
...original,
|
||||
tpostings: original.tpostings.slice(0, 3).map((posting) => ({
|
||||
...posting,
|
||||
pamount: [leaf]
|
||||
}))
|
||||
};
|
||||
return {
|
||||
// Inject a JSON number token, never an already-rounded JavaScript number.
|
||||
originalJson: ` \n${JSON.stringify(original, null, '\t').replace(
|
||||
'"__EXACT_NATIVE_MANTISSA__"',
|
||||
'900719925474099301e2'
|
||||
)}\n `,
|
||||
transactedJson: `\n${JSON.stringify(transacted, null, ' ')}\t`
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { Kysely } from 'kysely';
|
||||
import { afterEach, beforeEach, expect, it } from 'vitest';
|
||||
import {
|
||||
createReportDatabase,
|
||||
openReadOnlyReportDatabase,
|
||||
runMigrations,
|
||||
type ReportDatabase
|
||||
} from '../index.js';
|
||||
import {
|
||||
assertTransactionImportComplete,
|
||||
attachTransactedTransaction,
|
||||
insertNativeTransaction,
|
||||
readAccountTransactionBatch,
|
||||
readNativeTransaction,
|
||||
type StoredNativeTransaction
|
||||
} from './queries.js';
|
||||
import { nativeTransactionFixture, richNativeExportsFixture } from './native.fixture.js';
|
||||
|
||||
let directory: string;
|
||||
let path: string;
|
||||
const databases = new Set<Kysely<ReportDatabase>>();
|
||||
|
||||
async function createDatabase(): Promise<Kysely<ReportDatabase>> {
|
||||
const database = await createReportDatabase(path);
|
||||
databases.add(database);
|
||||
await runMigrations(database);
|
||||
return database;
|
||||
}
|
||||
|
||||
async function closeDatabase(database: Kysely<ReportDatabase>): Promise<void> {
|
||||
await database.destroy();
|
||||
databases.delete(database);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(join(tmpdir(), 'native-transactions-'));
|
||||
path = join(directory, 'reports.sqlite');
|
||||
});
|
||||
afterEach(async () => {
|
||||
try {
|
||||
await Promise.all([...databases].map((database) => database.destroy()));
|
||||
} finally {
|
||||
databases.clear();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('retains exact native exports and original posting order through completion and read-only reopen', async () => {
|
||||
const database = await createDatabase();
|
||||
const exports = richNativeExportsFixture();
|
||||
const nextJson = JSON.stringify(nativeTransactionFixture(8));
|
||||
const complete: StoredNativeTransaction = {
|
||||
nativeIndex: 7,
|
||||
date: '2026-05-01',
|
||||
status: 'Pending',
|
||||
...exports
|
||||
};
|
||||
await expect(assertTransactionImportComplete(database)).resolves.toBeUndefined();
|
||||
const writer = await database.startTransaction().execute();
|
||||
try {
|
||||
await insertNativeTransaction(writer, exports.originalJson);
|
||||
expect(await readNativeTransaction(writer, 7)).toEqual({
|
||||
...complete,
|
||||
transactedJson: null
|
||||
});
|
||||
expect(await readNativeTransaction(writer, 8)).toBeNull();
|
||||
await expect(assertTransactionImportComplete(writer)).rejects.toThrow();
|
||||
await insertNativeTransaction(writer, nextJson);
|
||||
|
||||
// The valued export omits a conversion posting; its absence must not rewrite the index.
|
||||
await attachTransactedTransaction(writer, exports.transactedJson);
|
||||
expect(
|
||||
await writer
|
||||
.selectFrom('transaction_postings')
|
||||
.selectAll()
|
||||
.where('native_transaction_index', '=', 7)
|
||||
.orderBy('posting_ordinal', 'asc')
|
||||
.execute()
|
||||
).toEqual([
|
||||
{
|
||||
native_transaction_index: 7,
|
||||
posting_ordinal: 0,
|
||||
account_name: 'assets:cash',
|
||||
posting_date: '2026-04-16',
|
||||
status: 'Cleared'
|
||||
},
|
||||
{
|
||||
native_transaction_index: 7,
|
||||
posting_ordinal: 1,
|
||||
account_name: 'expenses:office',
|
||||
posting_date: null,
|
||||
status: 'Unmarked'
|
||||
},
|
||||
{
|
||||
native_transaction_index: 7,
|
||||
posting_ordinal: 2,
|
||||
account_name: 'assets:cash',
|
||||
posting_date: '2026-04-16',
|
||||
status: 'Cleared'
|
||||
},
|
||||
{
|
||||
native_transaction_index: 7,
|
||||
posting_ordinal: 3,
|
||||
account_name: 'equity:conversion',
|
||||
posting_date: null,
|
||||
status: 'Unmarked'
|
||||
}
|
||||
]);
|
||||
await expect(assertTransactionImportComplete(writer)).rejects.toThrow();
|
||||
await attachTransactedTransaction(writer, nextJson);
|
||||
await assertTransactionImportComplete(writer);
|
||||
await writer.commit().execute();
|
||||
} finally {
|
||||
if (!writer.isCommitted && !writer.isRolledBack) await writer.rollback().execute();
|
||||
}
|
||||
await closeDatabase(database);
|
||||
|
||||
const reader = await openReadOnlyReportDatabase(path);
|
||||
if (!reader) throw new Error('Expected the current report schema.');
|
||||
databases.add(reader);
|
||||
expect(await readNativeTransaction(reader, 7)).toEqual(complete);
|
||||
expect(await readNativeTransaction(reader, 8)).toEqual({
|
||||
nativeIndex: 8,
|
||||
date: '2026-05-01',
|
||||
status: 'Pending',
|
||||
originalJson: nextJson,
|
||||
transactedJson: nextJson
|
||||
});
|
||||
expect(await readNativeTransaction(reader, 9)).toBeNull();
|
||||
expect(await readAccountTransactionBatch(reader, 'equity:conversion', 0)).toEqual([complete]);
|
||||
await expect(assertTransactionImportComplete(reader)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects unknown, duplicate and structurally mismatched native exports without replacing saved input', async () => {
|
||||
const database = await createDatabase();
|
||||
const exports = richNativeExportsFixture();
|
||||
const writer = await database.startTransaction().execute();
|
||||
try {
|
||||
await expect(attachTransactedTransaction(writer, exports.transactedJson)).rejects.toThrow();
|
||||
await insertNativeTransaction(writer, exports.originalJson);
|
||||
await expect(insertNativeTransaction(writer, exports.originalJson)).rejects.toThrow();
|
||||
const mismatch = exports.transactedJson.replace('"expenses:office"', '"expenses:other"');
|
||||
await expect(attachTransactedTransaction(writer, mismatch)).rejects.toThrow();
|
||||
expect(await readNativeTransaction(writer, 7)).toEqual({
|
||||
nativeIndex: 7,
|
||||
date: '2026-05-01',
|
||||
status: 'Pending',
|
||||
originalJson: exports.originalJson,
|
||||
transactedJson: null
|
||||
});
|
||||
await expect(assertTransactionImportComplete(writer)).rejects.toThrow();
|
||||
await attachTransactedTransaction(writer, exports.transactedJson);
|
||||
await expect(attachTransactedTransaction(writer, exports.transactedJson)).rejects.toThrow();
|
||||
expect(await readNativeTransaction(writer, 7)).toEqual({
|
||||
nativeIndex: 7,
|
||||
date: '2026-05-01',
|
||||
status: 'Pending',
|
||||
...exports
|
||||
});
|
||||
await assertTransactionImportComplete(writer);
|
||||
await writer.commit().execute();
|
||||
} finally {
|
||||
if (!writer.isCommitted && !writer.isRolledBack) await writer.rollback().execute();
|
||||
}
|
||||
});
|
||||
|
||||
it('fetches bounded native-index batches of whole transactions for literal binary account subtrees', async () => {
|
||||
const database = await createDatabase();
|
||||
const account = 'assets:Ca%_fé';
|
||||
const expected: StoredNativeTransaction[] = Array.from({ length: 53 }, (_, ordinal) => {
|
||||
const matching = ordinal % 2 === 0 ? account : `${account}:钱包`;
|
||||
const accounts =
|
||||
ordinal === 0
|
||||
? [...Array.from({ length: 50 }, () => matching), 'expenses:chunk-end']
|
||||
: [matching, 'expenses:counterpart', matching];
|
||||
const native = nativeTransactionFixture(101 + ordinal * 3, accounts);
|
||||
// Native index order, not transaction date or insertion order, owns the cursor.
|
||||
native.tdate = ordinal < 26 ? '2026-05-02' : '2026-05-01';
|
||||
return {
|
||||
nativeIndex: native.tindex,
|
||||
date: native.tdate,
|
||||
status: native.tstatus,
|
||||
originalJson: JSON.stringify(native),
|
||||
transactedJson: null
|
||||
};
|
||||
});
|
||||
const excludedAccounts = [
|
||||
'assets:ca%_fé',
|
||||
'assets:Ca%_fÉ',
|
||||
'assets:CaZZfé',
|
||||
'assets:Ca%xfé',
|
||||
`${account}ature`,
|
||||
`${account};钱包`
|
||||
];
|
||||
const writer = await database.startTransaction().execute();
|
||||
try {
|
||||
for (const stored of [...expected].reverse()) {
|
||||
await insertNativeTransaction(writer, stored.originalJson);
|
||||
}
|
||||
for (const [ordinal, excluded] of excludedAccounts.entries()) {
|
||||
const native = nativeTransactionFixture(1000 + ordinal, [excluded, 'expenses:counterpart']);
|
||||
await insertNativeTransaction(writer, JSON.stringify(native));
|
||||
}
|
||||
await writer.commit().execute();
|
||||
} finally {
|
||||
if (!writer.isCommitted && !writer.isRolledBack) await writer.rollback().execute();
|
||||
}
|
||||
|
||||
expect(await readAccountTransactionBatch(database, 'expenses:chunk-end', 0)).toEqual([
|
||||
expected[0]
|
||||
]);
|
||||
const resultRowCounts: number[] = [];
|
||||
const batchReader = database.withPlugin({
|
||||
transformQuery: ({ node }) => node,
|
||||
async transformResult({ result }) {
|
||||
resultRowCounts.push(result.rows.length);
|
||||
return result;
|
||||
}
|
||||
});
|
||||
const first = await readAccountTransactionBatch(batchReader, account, 0);
|
||||
expect(first).toEqual(expected.slice(0, 50));
|
||||
const last = await readAccountTransactionBatch(
|
||||
batchReader,
|
||||
account,
|
||||
first[first.length - 1].nativeIndex
|
||||
);
|
||||
expect(last).toEqual(expected.slice(50));
|
||||
expect(
|
||||
await readAccountTransactionBatch(batchReader, account, last[last.length - 1].nativeIndex)
|
||||
).toEqual([]);
|
||||
expect(Math.max(...resultRowCounts)).toBe(50);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
import {
|
||||
assertTransactionExportsMatch,
|
||||
decodeNativeTransaction,
|
||||
type NativeTransaction
|
||||
} from '@hledger-reports/hledger/transactions';
|
||||
import type { Kysely } from 'kysely';
|
||||
import type { ReportTransaction } from '../database.js';
|
||||
import type { ReportDatabase } from '../schema.js';
|
||||
|
||||
const postingInsertBatchSize = 50;
|
||||
const transactionBatchSize = 50;
|
||||
|
||||
export interface StoredNativeTransaction {
|
||||
/** Native tindex is meaningful only within this saved generation. */
|
||||
nativeIndex: number;
|
||||
date: string;
|
||||
status: string;
|
||||
/** Unvalued native print output, not source journal text. */
|
||||
originalJson: string;
|
||||
transactedJson: string | null;
|
||||
}
|
||||
|
||||
const storedTransactionColumns = [
|
||||
'native_index as nativeIndex',
|
||||
'transaction_date as date',
|
||||
'status',
|
||||
'original_json as originalJson',
|
||||
'transacted_json as transactedJson'
|
||||
] as const;
|
||||
|
||||
function assertTransactionIndexFields(transaction: NativeTransaction): void {
|
||||
if (
|
||||
!Number.isSafeInteger(transaction.tindex) ||
|
||||
transaction.tindex <= 0 ||
|
||||
typeof transaction.tdate !== 'string' ||
|
||||
typeof transaction.tstatus !== 'string'
|
||||
)
|
||||
throw new Error('Invalid native transaction index data.');
|
||||
for (const posting of transaction.tpostings) {
|
||||
if (
|
||||
!posting ||
|
||||
typeof posting !== 'object' ||
|
||||
Array.isArray(posting) ||
|
||||
typeof posting.paccount !== 'string' ||
|
||||
(posting.pdate !== null && typeof posting.pdate !== 'string') ||
|
||||
typeof posting.pstatus !== 'string'
|
||||
)
|
||||
throw new Error('Invalid native posting index data.');
|
||||
}
|
||||
}
|
||||
|
||||
export async function insertNativeTransaction(
|
||||
database: ReportTransaction,
|
||||
json: string
|
||||
): Promise<void> {
|
||||
const transaction = decodeNativeTransaction(json);
|
||||
assertTransactionIndexFields(transaction);
|
||||
await database
|
||||
.insertInto('transactions')
|
||||
.values({
|
||||
native_index: transaction.tindex,
|
||||
transaction_date: transaction.tdate,
|
||||
status: transaction.tstatus,
|
||||
original_json: json,
|
||||
transacted_json: null
|
||||
})
|
||||
.execute();
|
||||
for (let start = 0; start < transaction.tpostings.length; start += postingInsertBatchSize) {
|
||||
const postings = transaction.tpostings
|
||||
.slice(start, start + postingInsertBatchSize)
|
||||
.map((posting, index) => ({
|
||||
native_transaction_index: transaction.tindex,
|
||||
posting_ordinal: start + index,
|
||||
account_name: posting.paccount,
|
||||
posting_date: posting.pdate,
|
||||
status: posting.pstatus
|
||||
}));
|
||||
await database.insertInto('transaction_postings').values(postings).execute();
|
||||
}
|
||||
}
|
||||
|
||||
export async function attachTransactedTransaction(
|
||||
database: ReportTransaction,
|
||||
json: string
|
||||
): Promise<void> {
|
||||
const transacted = decodeNativeTransaction(json);
|
||||
assertTransactionIndexFields(transacted);
|
||||
const stored = await database
|
||||
.selectFrom('transactions')
|
||||
.select(['original_json', 'transacted_json'])
|
||||
.where('native_index', '=', transacted.tindex)
|
||||
.executeTakeFirst();
|
||||
if (!stored) throw new Error('Missing original native transaction.');
|
||||
if (stored.transacted_json !== null)
|
||||
throw new Error('Transacted native transaction is already attached.');
|
||||
const original = decodeNativeTransaction(stored.original_json);
|
||||
assertTransactionExportsMatch(original, transacted);
|
||||
const result = await database
|
||||
.updateTable('transactions')
|
||||
.set({ transacted_json: json })
|
||||
.where('native_index', '=', transacted.tindex)
|
||||
.where('transacted_json', 'is', null)
|
||||
.executeTakeFirstOrThrow();
|
||||
if (result.numUpdatedRows !== 1n)
|
||||
throw new Error('Transacted native transaction was not attached.');
|
||||
}
|
||||
|
||||
export async function readNativeTransaction(
|
||||
database: Kysely<ReportDatabase>,
|
||||
nativeIndex: number
|
||||
): Promise<StoredNativeTransaction | null> {
|
||||
if (!Number.isSafeInteger(nativeIndex) || nativeIndex <= 0)
|
||||
throw new Error('Invalid native transaction index.');
|
||||
const stored = await database
|
||||
.selectFrom('transactions')
|
||||
.select(storedTransactionColumns)
|
||||
.where('native_index', '=', nativeIndex)
|
||||
.executeTakeFirst();
|
||||
return stored ?? null;
|
||||
}
|
||||
|
||||
export async function readAccountTransactionBatch(
|
||||
database: Kysely<ReportDatabase>,
|
||||
accountName: string,
|
||||
afterNativeIndex: number
|
||||
): Promise<StoredNativeTransaction[]> {
|
||||
if (
|
||||
typeof accountName !== 'string' ||
|
||||
!accountName.trim() ||
|
||||
!Number.isSafeInteger(afterNativeIndex) ||
|
||||
afterNativeIndex < 0
|
||||
)
|
||||
throw new Error('Invalid account transaction cursor.');
|
||||
// In BINARY order, [name + ':', name + ';') selects colon-delimited descendants.
|
||||
const matchingIndexes = database
|
||||
.selectFrom('transaction_postings')
|
||||
.select('native_transaction_index')
|
||||
.distinct()
|
||||
.where('native_transaction_index', '>', afterNativeIndex)
|
||||
.where((expression) =>
|
||||
expression.or([
|
||||
expression('account_name', '=', accountName),
|
||||
expression.and([
|
||||
expression('account_name', '>=', `${accountName}:`),
|
||||
expression('account_name', '<', `${accountName};`)
|
||||
])
|
||||
])
|
||||
)
|
||||
.orderBy('native_transaction_index', 'asc')
|
||||
.limit(transactionBatchSize);
|
||||
return database
|
||||
.selectFrom('transactions')
|
||||
.select(storedTransactionColumns)
|
||||
.where('native_index', 'in', matchingIndexes)
|
||||
.orderBy('native_index', 'asc')
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function assertTransactionImportComplete(
|
||||
database: Kysely<ReportDatabase>
|
||||
): Promise<void> {
|
||||
const unfilled = await database
|
||||
.selectFrom('transactions')
|
||||
.select('native_index')
|
||||
.where('transacted_json', 'is', null)
|
||||
.limit(1)
|
||||
.executeTakeFirst();
|
||||
if (unfilled) throw new Error('Native transaction import is incomplete.');
|
||||
}
|
||||
@@ -8,6 +8,11 @@
|
||||
"development": "./src/index.ts",
|
||||
"types": "./dist/src/index.d.ts",
|
||||
"import": "./dist/src/index.js"
|
||||
},
|
||||
"./transactions": {
|
||||
"development": "./src/transactions/index.ts",
|
||||
"types": "./dist/src/transactions/index.d.ts",
|
||||
"import": "./dist/src/transactions/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
|
||||
@@ -6,131 +6,42 @@ import {
|
||||
organizeAccountBalance,
|
||||
type PreparedBalance
|
||||
} from '../accounts/balances.js';
|
||||
import { journalMetadata } from '../journal-stats.js';
|
||||
import type { InvestmentMetadata } from '../transactions/investment.js';
|
||||
import type { PreparationMetadata, QualifiedAccount } from './metadata.js';
|
||||
|
||||
export interface PreparedAccountSummary {
|
||||
name: string;
|
||||
access: 'ready' | 'ambiguous';
|
||||
export interface PreparedAccountSummary extends QualifiedAccount {
|
||||
rowCount: number | null;
|
||||
summary: PreparedBalance;
|
||||
}
|
||||
|
||||
const accountMarker = '\n__HLEDGER_PREPARED_ACCOUNTS__\n';
|
||||
const discoveryEnd = '\n__HLEDGER_PREPARED_DISCOVERY_END__\n';
|
||||
const findMarker = '\n__HLEDGER_PREPARED_FIND__\n';
|
||||
export const registersMarker = '\n__HLEDGER_PREPARED_REGISTERS__\n';
|
||||
export const accountMarkers = {
|
||||
originalEnd: '\n__HLEDGER_PREPARED_ORIGINAL_END__\n',
|
||||
transactedEnd: '\n__HLEDGER_PREPARED_TRANSACTED_END__\n',
|
||||
registerEnd: '\n__HLEDGER_PREPARED_REGISTER_END__\n'
|
||||
} as const;
|
||||
const roleQueries = [
|
||||
['assets', 'A', '\n__HLEDGER_PREPARED_ASSETS__\n'],
|
||||
['cash', 'C', '\n__HLEDGER_PREPARED_CASH__\n'],
|
||||
['liabilities', 'L', '\n__HLEDGER_PREPARED_LIABILITIES__\n'],
|
||||
['income', 'R', '\n__HLEDGER_PREPARED_INCOME__\n'],
|
||||
['expenses', 'X', '\n__HLEDGER_PREPARED_EXPENSES__\n'],
|
||||
['gains', 'G', '\n__HLEDGER_PREPARED_GAINS__\n']
|
||||
] as const;
|
||||
const lotsMarker = '\n__HLEDGER_PREPARED_LOT_COMMODITIES__\n';
|
||||
const failureMessage = 'Invalid prepared report batch.';
|
||||
|
||||
function echo(marker: string): string[] {
|
||||
return ['--', 'echo', marker.trimEnd()];
|
||||
}
|
||||
|
||||
function accountNames(text: string): string[] {
|
||||
if (text === '') return [];
|
||||
if (!text.endsWith('\n')) throw new Error(failureMessage);
|
||||
const names = text
|
||||
.slice(0, -1)
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
if (!line.startsWith('account ')) throw new Error(failureMessage);
|
||||
const name = line.slice('account '.length);
|
||||
// Names are identities, never paths. Reject controls and unrepresentable
|
||||
// identities rather than normalizing native whitespace or Unicode.
|
||||
if (!name.trim() || /\p{Cc}/u.test(name)) {
|
||||
throw new Error(failureMessage);
|
||||
}
|
||||
return name;
|
||||
});
|
||||
if (new Set(names).size !== names.length) throw new Error(failureMessage);
|
||||
return names;
|
||||
}
|
||||
|
||||
function commodityNames(text: string): ReadonlySet<string> {
|
||||
if (text === '') return new Set();
|
||||
if (!text.endsWith('\n')) throw new Error(failureMessage);
|
||||
const names = text.slice(0, -1).split('\n');
|
||||
// Commodities emits raw symbols, not quoted commodity directives.
|
||||
if (names.some((name) => /\p{Cc}/u.test(name)) || new Set(names).size !== names.length)
|
||||
throw new Error(failureMessage);
|
||||
return new Set(names);
|
||||
}
|
||||
|
||||
function splitLast(text: string, marker: string): [string, string] {
|
||||
const index = text.lastIndexOf(marker);
|
||||
if (index < 0) throw new Error(failureMessage);
|
||||
return [text.slice(0, index), text.slice(index + marker.length)];
|
||||
}
|
||||
|
||||
function escapeAccountName(name: string): string {
|
||||
return name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
export function planDiscoveryCommands(inputFlags: readonly string[]): string[] {
|
||||
return [
|
||||
...inputFlags,
|
||||
'run',
|
||||
'--',
|
||||
'stats',
|
||||
...echo(accountMarker),
|
||||
'--',
|
||||
'accounts',
|
||||
'--flat',
|
||||
'--directives',
|
||||
...echo(discoveryEnd)
|
||||
];
|
||||
}
|
||||
|
||||
export function decodeDiscovery(text: string) {
|
||||
if (!text.endsWith(discoveryEnd)) throw new Error(failureMessage);
|
||||
const [stats, namesText] = splitLast(text.slice(0, -discoveryEnd.length), accountMarker);
|
||||
const names = accountNames(namesText).sort();
|
||||
const discovery = journalMetadata(stats);
|
||||
return { names, discovery };
|
||||
}
|
||||
|
||||
export function planPreparedReportCommands(
|
||||
inputFlags: readonly string[],
|
||||
discovery: ReturnType<typeof journalMetadata>,
|
||||
names: readonly string[]
|
||||
metadata: PreparationMetadata
|
||||
) {
|
||||
const overviewPlan = planOverviewReports(discovery);
|
||||
const overviewPlan = planOverviewReports(metadata.journal);
|
||||
const args = [
|
||||
...inputFlags,
|
||||
'run',
|
||||
'--',
|
||||
...overviewPlan.commands,
|
||||
...planAccountBalances(discovery),
|
||||
...echo(accountMarker),
|
||||
'--',
|
||||
'accounts',
|
||||
'--flat',
|
||||
'--directives'
|
||||
...planAccountBalances(metadata.journal),
|
||||
...echo(registersMarker)
|
||||
];
|
||||
for (const name of names) {
|
||||
const exactAccountPattern = `^${escapeAccountName(name)}$`;
|
||||
args.push(...echo(findMarker), '--', 'accounts', '--find', '--directives', exactAccountPattern);
|
||||
}
|
||||
for (const [, type, marker] of roleQueries) {
|
||||
args.push(...echo(marker), '--', 'accounts', '--flat', '--directives', `type:${type}`);
|
||||
}
|
||||
args.push(...echo(lotsMarker), '--', 'commodities', 'tag:lots');
|
||||
args.push(...echo(registersMarker));
|
||||
for (const name of names) {
|
||||
for (const { name } of metadata.accounts) {
|
||||
const escapedName = escapeAccountName(name);
|
||||
const exactAccountPattern = `^${escapedName}$`;
|
||||
// Print's acct query is case-insensitive; exact canonical membership is
|
||||
@@ -169,60 +80,23 @@ export function planPreparedReportCommands(
|
||||
|
||||
export function decodePreparedPrefix(
|
||||
prefix: string,
|
||||
names: string[],
|
||||
discovery: ReturnType<typeof journalMetadata>,
|
||||
metadata: PreparationMetadata,
|
||||
overviewPlan: ReturnType<typeof planOverviewReports>
|
||||
) {
|
||||
// These native query sets are part of the same financial prefix and
|
||||
// cached journal, never inferred from account names or inherited posting tags.
|
||||
const [beforeLots, lots] = splitLast(prefix, lotsMarker);
|
||||
prefix = beforeLots;
|
||||
const metadata: InvestmentMetadata = {
|
||||
assets: new Set(),
|
||||
cash: new Set(),
|
||||
liabilities: new Set(),
|
||||
income: new Set(),
|
||||
expenses: new Set(),
|
||||
gains: new Set(),
|
||||
lotCommodities: commodityNames(lots)
|
||||
};
|
||||
for (const [role, , marker] of [...roleQueries].reverse()) {
|
||||
const [before, text] = splitLast(prefix, marker);
|
||||
metadata[role] = new Set(accountNames(text));
|
||||
prefix = before;
|
||||
}
|
||||
const canonical: string[] = [];
|
||||
for (let index = names.length - 1; index >= 0; index--) {
|
||||
const [before, found] = splitLast(prefix, findMarker);
|
||||
const matches = accountNames(found);
|
||||
if (matches.length !== 1) throw new Error(failureMessage);
|
||||
canonical.unshift(matches[0]);
|
||||
prefix = before;
|
||||
}
|
||||
const [financial, actualNames] = splitLast(prefix, accountMarker);
|
||||
const actual = accountNames(actualNames).sort();
|
||||
if (actual.length !== names.length || actual.some((name, index) => name !== names[index])) {
|
||||
throw new Error(failureMessage);
|
||||
}
|
||||
const accountReports = decodeAccountBalances(financial, discovery.asOf);
|
||||
const accountReports = decodeAccountBalances(prefix, metadata.journal.asOf);
|
||||
const overview = organizeOverviewData(
|
||||
decodeOverviewReports(accountReports.overviewBatch, discovery, overviewPlan)
|
||||
decodeOverviewReports(accountReports.overviewBatch, metadata.journal, overviewPlan)
|
||||
);
|
||||
// Native --find defines equivalence; JS case folding does not.
|
||||
const counts = new Map<string, number>();
|
||||
for (const name of canonical) counts.set(name, (counts.get(name) ?? 0) + 1);
|
||||
const accounts: PreparedAccountSummary[] = names.map((name, index) => {
|
||||
const nativeName = canonical[index];
|
||||
const access = nativeName !== name || counts.get(nativeName) !== 1 ? 'ambiguous' : 'ready';
|
||||
const accounts: PreparedAccountSummary[] = metadata.accounts.map(({ name, access }) => {
|
||||
const rowCount = access === 'ambiguous' ? null : 0;
|
||||
const summary = organizeAccountBalance(
|
||||
name,
|
||||
access,
|
||||
discovery.asOf,
|
||||
discovery.baseCurrency,
|
||||
metadata.journal.asOf,
|
||||
metadata.journal.baseCurrency,
|
||||
accountReports
|
||||
);
|
||||
return { name, access, rowCount, summary };
|
||||
});
|
||||
return { overview, accounts, metadata };
|
||||
return { overview, accounts };
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { readHledger } from '../process.js';
|
||||
import { readPreparationMetadata } from './metadata.js';
|
||||
|
||||
vi.mock('../process.js', () => ({ readHledger: vi.fn() }));
|
||||
|
||||
const read = vi.mocked(readHledger);
|
||||
const stats = 'Base currency : USD\nLast txn : none\nTxns : 0\nTxns span : to\n';
|
||||
|
||||
beforeEach(() => {
|
||||
read.mockReset();
|
||||
});
|
||||
|
||||
describe('readPreparationMetadata', () => {
|
||||
it('keeps native types, canonical access and raw lot commodities independent', async () => {
|
||||
const inputFlags = ['-f', 'fictional.journal'];
|
||||
const options = { signal: new AbortController().signal };
|
||||
read
|
||||
.mockResolvedValueOnce(stats)
|
||||
.mockResolvedValueOnce(
|
||||
'holdings\nholdings:child\nliabilities\nequity\nspending\nrevenue\nrevenue:realized\nconversion\nuntyped\nWallet\nwallet\nalias=資産[USD].*\n'
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
'holdings ; type: C\nholdings:child ; type: C\nliabilities ; type: L\nequity ; type: E\nspending ; type: X\nrevenue ; type: R\nrevenue:realized ; type: G\nconversion ; type: V\nuntyped\nWallet ; type: C\nwallet ; type: A\nalias=資産[USD].* ; type: A\n'
|
||||
)
|
||||
.mockResolvedValueOnce(' USD \n\n資産[USD].*\n__HLEDGER_METADATA_END__\n; type: C\n')
|
||||
.mockResolvedValueOnce(
|
||||
'Wallet\nalias=資産[USD].*\nconversion\nequity\nholdings\nholdings:child\nliabilities\nrevenue\nrevenue:realized\nspending\nuntyped\nWallet\n'
|
||||
);
|
||||
|
||||
const metadata = await readPreparationMetadata(inputFlags, options);
|
||||
|
||||
expect(metadata).toEqual({
|
||||
journal: {
|
||||
asOf: null,
|
||||
baseCurrency: 'USD',
|
||||
journalCoverage: { transactionCount: 0, firstDate: null, lastDate: null }
|
||||
},
|
||||
accounts: [
|
||||
{ name: 'Wallet', access: 'ambiguous' },
|
||||
{ name: 'alias=資産[USD].*', access: 'ready' },
|
||||
{ name: 'conversion', access: 'ready' },
|
||||
{ name: 'equity', access: 'ready' },
|
||||
{ name: 'holdings', access: 'ready' },
|
||||
{ name: 'holdings:child', access: 'ready' },
|
||||
{ name: 'liabilities', access: 'ready' },
|
||||
{ name: 'revenue', access: 'ready' },
|
||||
{ name: 'revenue:realized', access: 'ready' },
|
||||
{ name: 'spending', access: 'ready' },
|
||||
{ name: 'untyped', access: 'ready' },
|
||||
{ name: 'wallet', access: 'ambiguous' }
|
||||
],
|
||||
investment: {
|
||||
assets: new Set(['holdings', 'holdings:child', 'Wallet', 'wallet', 'alias=資産[USD].*']),
|
||||
cash: new Set(['holdings', 'holdings:child', 'Wallet']),
|
||||
liabilities: new Set(['liabilities']),
|
||||
income: new Set(['revenue', 'revenue:realized']),
|
||||
expenses: new Set(['spending']),
|
||||
gains: new Set(['revenue:realized']),
|
||||
lotCommodities: new Set([
|
||||
' USD ',
|
||||
'',
|
||||
'資産[USD].*',
|
||||
'__HLEDGER_METADATA_END__',
|
||||
'; type: C'
|
||||
])
|
||||
}
|
||||
});
|
||||
expect(read).toHaveBeenCalledTimes(5);
|
||||
for (const [, actualOptions] of read.mock.calls) {
|
||||
expect(actualOptions?.signal).toBe(options.signal);
|
||||
}
|
||||
});
|
||||
|
||||
it('uses exact native lookup identities without JavaScript folding and batches literal patterns', async () => {
|
||||
const names = ['wrong', 'z[USD].*', 'İ', 'i\u0307'].join('\n') + '\n';
|
||||
read
|
||||
.mockResolvedValueOnce(stats)
|
||||
.mockResolvedValueOnce(names)
|
||||
.mockResolvedValueOnce(names)
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('i\u0307\nunexpected\nz[USD].*\nİ\n');
|
||||
const metadata = await readPreparationMetadata([], {});
|
||||
expect(metadata.accounts).toEqual([
|
||||
{ name: 'i\u0307', access: 'ready' },
|
||||
{ name: 'wrong', access: 'ambiguous' },
|
||||
{ name: 'z[USD].*', access: 'ready' },
|
||||
{ name: 'İ', access: 'ready' }
|
||||
]);
|
||||
const expectedFindCommands = [
|
||||
['--', 'accounts', '--find', '^i\u0307$'],
|
||||
['--', 'accounts', '--find', '^wrong$'],
|
||||
['--', 'accounts', '--find', '^z\\[USD\\]\\.\\*$'],
|
||||
['--', 'accounts', '--find', '^İ$']
|
||||
];
|
||||
expect(read.mock.calls.map(([args]) => args)).toEqual([
|
||||
['stats'],
|
||||
['accounts', '--flat'],
|
||||
['accounts', '--flat', '--types'],
|
||||
['commodities', 'tag:lots'],
|
||||
['run', ...expectedFindCommands.flat()]
|
||||
]);
|
||||
});
|
||||
|
||||
it('pairs typed lines with exact bare identities, including trailing spaces and type-like suffixes', async () => {
|
||||
read
|
||||
.mockResolvedValueOnce(stats)
|
||||
.mockResolvedValueOnce('assets:cash \nassets:literal ; type: C\n')
|
||||
.mockResolvedValueOnce(
|
||||
'assets:cash ; type: C\nassets:literal ; type: C ; type: A\n'
|
||||
)
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('assets:cash \nassets:literal ; type: C\n');
|
||||
|
||||
const metadata = await readPreparationMetadata([], {});
|
||||
|
||||
expect(metadata.accounts).toEqual([
|
||||
{ name: 'assets:cash ', access: 'ready' },
|
||||
{ name: 'assets:literal ; type: C', access: 'ready' }
|
||||
]);
|
||||
expect(metadata.investment.assets).toEqual(
|
||||
new Set(['assets:cash ', 'assets:literal ; type: C'])
|
||||
);
|
||||
expect(metadata.investment.cash).toEqual(new Set(['assets:cash ']));
|
||||
});
|
||||
|
||||
it('rejects a typed catalog that does not match the bare identities before reading further', async () => {
|
||||
read
|
||||
.mockResolvedValueOnce(stats)
|
||||
.mockResolvedValueOnce('assets\n')
|
||||
.mockResolvedValueOnce('liabilities ; type: L\n');
|
||||
|
||||
await expect(readPreparationMetadata([], {})).rejects.toThrow(
|
||||
'Invalid native preparation metadata.'
|
||||
);
|
||||
expect(read).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('still reads lot commodities for an empty account catalog without running find', async () => {
|
||||
read
|
||||
.mockResolvedValueOnce(stats)
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('\nLOT\n');
|
||||
|
||||
const metadata = await readPreparationMetadata([], {});
|
||||
|
||||
expect(metadata.accounts).toEqual([]);
|
||||
expect(metadata.investment).toEqual({
|
||||
assets: new Set(),
|
||||
cash: new Set(),
|
||||
liabilities: new Set(),
|
||||
income: new Set(),
|
||||
expenses: new Set(),
|
||||
gains: new Set(),
|
||||
lotCommodities: new Set(['', 'LOT'])
|
||||
});
|
||||
expect(read.mock.calls.map(([args]) => args)).toEqual([
|
||||
['stats'],
|
||||
['accounts', '--flat'],
|
||||
['accounts', '--flat', '--types'],
|
||||
['commodities', 'tag:lots']
|
||||
]);
|
||||
});
|
||||
|
||||
it('waits for the active metadata read and starts no later read after cancellation', async () => {
|
||||
const controller = new AbortController();
|
||||
const reason = new Error('Preparation cancelled');
|
||||
let releaseStats!: (text: string) => void;
|
||||
read.mockReturnValueOnce(
|
||||
new Promise<string>((resolve) => {
|
||||
releaseStats = resolve;
|
||||
})
|
||||
);
|
||||
const outcome = readPreparationMetadata([], { signal: controller.signal }).catch(
|
||||
(error) => error
|
||||
);
|
||||
try {
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
controller.abort(reason);
|
||||
releaseStats(stats);
|
||||
}
|
||||
expect(await outcome).toBe(reason);
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import { journalMetadata } from '../journal-stats.js';
|
||||
import { readHledger } from '../process.js';
|
||||
import type { InvestmentMetadata } from '../transactions/investment.js';
|
||||
|
||||
export interface QualifiedAccount {
|
||||
name: string;
|
||||
access: 'ready' | 'ambiguous';
|
||||
}
|
||||
|
||||
export interface PreparationMetadata {
|
||||
journal: ReturnType<typeof journalMetadata>;
|
||||
accounts: QualifiedAccount[];
|
||||
investment: InvestmentMetadata;
|
||||
}
|
||||
|
||||
const failureMessage = 'Invalid native preparation metadata.';
|
||||
|
||||
function nativeLines(text: string): string[] {
|
||||
if (text === '') return [];
|
||||
if (!text.endsWith('\n')) throw new Error(failureMessage);
|
||||
return text.slice(0, -1).split('\n');
|
||||
}
|
||||
|
||||
function accountNames(text: string): string[] {
|
||||
const names = nativeLines(text);
|
||||
if (
|
||||
names.some((name) => !name.trim() || /\p{Cc}/u.test(name)) ||
|
||||
new Set(names).size !== names.length
|
||||
)
|
||||
throw new Error(failureMessage);
|
||||
return names;
|
||||
}
|
||||
|
||||
function typedAccounts(names: readonly string[], text: string) {
|
||||
const lines = nativeLines(text);
|
||||
if (lines.length !== names.length) throw new Error(failureMessage);
|
||||
return names.map((name, index) => {
|
||||
const line = lines[index];
|
||||
if (line === name) return { name, type: null };
|
||||
if (!line.startsWith(name)) throw new Error(failureMessage);
|
||||
// Native --types pads names. Remove the known bare identity, never trim it.
|
||||
const type = /^ {4,}; type: ([ACLERXGV])$/.exec(line.slice(name.length));
|
||||
if (!type) throw new Error(failureMessage);
|
||||
return { name, type: type[1] };
|
||||
});
|
||||
}
|
||||
|
||||
function lotCommodities(text: string): ReadonlySet<string> {
|
||||
const symbols = nativeLines(text);
|
||||
if (symbols.some((symbol) => /\p{Cc}/u.test(symbol)) || new Set(symbols).size !== symbols.length)
|
||||
throw new Error(failureMessage);
|
||||
return new Set(symbols);
|
||||
}
|
||||
|
||||
function investmentMetadata(
|
||||
accounts: ReturnType<typeof typedAccounts>,
|
||||
lotSymbols: ReadonlySet<string>
|
||||
): InvestmentMetadata {
|
||||
const assets = new Set<string>();
|
||||
const cash = new Set<string>();
|
||||
const liabilities = new Set<string>();
|
||||
const income = new Set<string>();
|
||||
const expenses = new Set<string>();
|
||||
const gains = new Set<string>();
|
||||
for (const { name, type } of accounts) {
|
||||
switch (type) {
|
||||
case 'C':
|
||||
cash.add(name);
|
||||
assets.add(name);
|
||||
break;
|
||||
case 'A':
|
||||
assets.add(name);
|
||||
break;
|
||||
case 'L':
|
||||
liabilities.add(name);
|
||||
break;
|
||||
case 'G':
|
||||
gains.add(name);
|
||||
income.add(name);
|
||||
break;
|
||||
case 'R':
|
||||
income.add(name);
|
||||
break;
|
||||
case 'X':
|
||||
expenses.add(name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return {
|
||||
assets,
|
||||
cash,
|
||||
liabilities,
|
||||
income,
|
||||
expenses,
|
||||
gains,
|
||||
lotCommodities: lotSymbols
|
||||
};
|
||||
}
|
||||
|
||||
function canonicalLookups(names: readonly string[]): string[] {
|
||||
const args = ['run'];
|
||||
for (const name of names) {
|
||||
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
args.push('--', 'accounts', '--find', `^${escaped}$`);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function qualifyAccounts(names: readonly string[], text: string): QualifiedAccount[] {
|
||||
// Each native --find produces exactly one bare name, in command order.
|
||||
const found = nativeLines(text);
|
||||
if (found.length !== names.length || found.some((name) => !name.trim() || /\p{Cc}/u.test(name)))
|
||||
throw new Error(failureMessage);
|
||||
const counts = new Map<string, number>();
|
||||
for (const name of found) counts.set(name, (counts.get(name) ?? 0) + 1);
|
||||
return names.map((name, index) => {
|
||||
let access: QualifiedAccount['access'] = 'ambiguous';
|
||||
if (found[index] === name && counts.get(name) === 1) access = 'ready';
|
||||
return { name, access };
|
||||
});
|
||||
}
|
||||
|
||||
export async function readPreparationMetadata(
|
||||
inputFlags: readonly string[],
|
||||
options: { signal?: AbortSignal }
|
||||
): Promise<PreparationMetadata> {
|
||||
async function read(command: readonly string[]): Promise<string> {
|
||||
options.signal?.throwIfAborted();
|
||||
return readHledger([...inputFlags, ...command], options);
|
||||
}
|
||||
const journal = journalMetadata(await read(['stats']));
|
||||
const names = accountNames(await read(['accounts', '--flat']));
|
||||
const typed = typedAccounts(names, await read(['accounts', '--flat', '--types']));
|
||||
const lotSymbols = lotCommodities(await read(['commodities', 'tag:lots']));
|
||||
const orderedNames = [...names].sort();
|
||||
const accounts = orderedNames.length
|
||||
? qualifyAccounts(orderedNames, await read(canonicalLookups(orderedNames)))
|
||||
: [];
|
||||
options.signal?.throwIfAborted();
|
||||
const investment = investmentMetadata(typed, lotSymbols);
|
||||
return { journal, accounts, investment };
|
||||
}
|
||||
@@ -35,7 +35,14 @@ function collectRows() {
|
||||
}
|
||||
|
||||
it('joins native account context, original annotations and transacted values', async () => {
|
||||
const journal = await createJournal(cashJournal);
|
||||
const journal = await createJournal(`${cashJournal}
|
||||
account liquid ; type: C
|
||||
account liquid:daily
|
||||
|
||||
2026-04-16 Inherited cash role
|
||||
liquid:daily 1.00 USD
|
||||
equity:opening -1.00 USD
|
||||
`);
|
||||
const collected = collectRows();
|
||||
const prepared = await prepareReports(journal, collected);
|
||||
const rows = collected.rowsByAccount.get('assets:Cash')!;
|
||||
@@ -55,6 +62,14 @@ it('joins native account context, original annotations and transacted values', a
|
||||
],
|
||||
transactedAmounts: [{ commodity: 'USD', quantity: '10' }]
|
||||
});
|
||||
const inheritedCash = collected.rowsByAccount.get('liquid:daily')![0];
|
||||
expect(inheritedCash.investment.cash).toEqual({
|
||||
direction: 'in',
|
||||
change: [{ commodity: 'USD', quantity: '1.00' }],
|
||||
balance: [{ commodity: 'USD', quantity: '1.00' }],
|
||||
unclassifiedChange: [],
|
||||
unclassifiedBalance: []
|
||||
});
|
||||
for (const account of prepared.accounts) {
|
||||
if (account.access === 'ready') {
|
||||
expect(collected.rowsByAccount.get(account.name)).toHaveLength(account.rowCount!);
|
||||
|
||||
@@ -35,16 +35,7 @@ afterEach(async () => {
|
||||
|
||||
const versionOutput = `hledger ${supportedHledgerVersion}, SECRET banner\n`;
|
||||
const journalStats = 'Base currency : none\nLast txn : none\nTxns : 0\nTxns span : to\n';
|
||||
const accountCatalog = '\n__HLEDGER_PREPARED_ACCOUNTS__\naccount assets:cash\n';
|
||||
const discoveryOutput = `${journalStats}${accountCatalog}\n__HLEDGER_PREPARED_DISCOVERY_END__\n`;
|
||||
const rolePrefix =
|
||||
['ASSETS', 'CASH', 'LIABILITIES', 'INCOME', 'EXPENSES', 'GAINS']
|
||||
.map(
|
||||
(role) =>
|
||||
`\n__HLEDGER_PREPARED_${role}__\n${role === 'ASSETS' || role === 'CASH' ? 'account assets:cash\n' : ''}`
|
||||
)
|
||||
.join('') + '\n__HLEDGER_PREPARED_LOT_COMMODITIES__\n';
|
||||
const financialPrefix = `${journalStats}${accountCatalog}\n__HLEDGER_PREPARED_FIND__\naccount assets:cash\n${rolePrefix}\n__HLEDGER_PREPARED_REGISTERS__\n`;
|
||||
const financialPrefix = `${journalStats}\n__HLEDGER_PREPARED_REGISTERS__\n`;
|
||||
const originalEnd = '\n\n__HLEDGER_PREPARED_ORIGINAL_END__\n';
|
||||
const transactedEnd = '\n\n__HLEDGER_PREPARED_TRANSACTED_END__\n';
|
||||
const registerEnd = '\n\n__HLEDGER_PREPARED_REGISTER_END__\n';
|
||||
@@ -96,12 +87,20 @@ function queueChildOutput(output: string) {
|
||||
});
|
||||
}
|
||||
|
||||
function queuePreparationMetadata() {
|
||||
queueChildOutput(journalStats);
|
||||
queueChildOutput('assets:cash\n');
|
||||
queueChildOutput('assets:cash ; type: C\n');
|
||||
queueChildOutput('');
|
||||
queueChildOutput('assets:cash\n');
|
||||
}
|
||||
|
||||
function mockPreparationProcesses(
|
||||
output: string,
|
||||
{ exitCode = 0, keepOpen = false }: { exitCode?: number; keepOpen?: boolean } = {}
|
||||
): ChildProcessWithoutNullStreams {
|
||||
queueChildOutput(versionOutput);
|
||||
queueChildOutput(discoveryOutput);
|
||||
queuePreparationMetadata();
|
||||
const child = startFixtureChild(
|
||||
`process.stdin.pipe(process.stdout,{end:false});process.stdin.on('end',()=>process.stdout.write('',()=>{${keepOpen ? 'setInterval(()=>{},1000)' : `process.exit(${exitCode})`}}));`
|
||||
);
|
||||
@@ -138,6 +137,14 @@ describe('preparation completion and failure', () => {
|
||||
});
|
||||
expect(result.overview.preparedAt).toBe('2026-01-01T00:02:30.000Z');
|
||||
expect(result.overview.hledgerVersion).toBe(supportedHledgerVersion);
|
||||
expect(result.accounts).toMatchObject([
|
||||
{ name: 'assets:cash', access: 'ready', rowCount: 1, summary: { status: 'undated' } }
|
||||
]);
|
||||
expect(processMock.spawn).toHaveBeenCalledTimes(7);
|
||||
const inputFlags = ['--no-conf', '--lang=en', '--strict', '-f', '/private'];
|
||||
for (const [, args] of processMock.spawn.mock.calls.slice(1)) {
|
||||
expect(args.slice(0, inputFlags.length)).toEqual(inputFlags);
|
||||
}
|
||||
expect(JSON.stringify(result)).not.toContain('SECRET');
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
@@ -229,7 +236,7 @@ describe('preparation completion and failure', () => {
|
||||
signal: controller.signal,
|
||||
async writeRows() {}
|
||||
});
|
||||
await vi.waitFor(() => expect(processMock.spawn).toHaveBeenCalledTimes(3));
|
||||
await vi.waitFor(() => expect(processMock.spawn).toHaveBeenCalledTimes(7));
|
||||
controller.abort();
|
||||
await expect(result).rejects.toThrow();
|
||||
expect(child.signalCode).toBe('SIGKILL');
|
||||
@@ -257,17 +264,9 @@ describe('preparation completion and failure', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('batch identity and framing', () => {
|
||||
it('rejects a changed account catalog instead of mixing discovery and financial views', async () => {
|
||||
await expectRejected(
|
||||
`${accountExports.replace('account assets:cash\n', 'account assets:other\n')}[]${registerEnd}`
|
||||
);
|
||||
});
|
||||
|
||||
it('does not confuse stats-path markers with protocol boundaries', async () => {
|
||||
queueChildOutput(versionOutput);
|
||||
queueChildOutput(`Main file : /private${accountCatalog}part\n${discoveryOutput}`);
|
||||
queueChildOutput(
|
||||
describe('financial framing and metadata qualification', () => {
|
||||
it('does not confuse financial stats-path markers with protocol boundaries', async () => {
|
||||
mockPreparationProcesses(
|
||||
`Main file : /private\n__HLEDGER_PREPARED_REGISTERS__\npart\n${accountExports}[]${registerEnd}`
|
||||
);
|
||||
const result = await prepareReports('/private', { async writeRows() {} });
|
||||
@@ -282,8 +281,14 @@ describe('batch identity and framing', () => {
|
||||
|
||||
it('rejects unsafe account identities before starting financial execution', async () => {
|
||||
queueChildOutput(versionOutput);
|
||||
queueChildOutput(discoveryOutput.replace('assets:cash', 'assets:\tSECRET'));
|
||||
await expect(prepareReports('/private', { async writeRows() {} })).rejects.toThrow();
|
||||
expect(processMock.spawn).toHaveBeenCalledTimes(2);
|
||||
queueChildOutput(journalStats);
|
||||
queueChildOutput('assets:\tSECRET\n');
|
||||
const error = await prepareReports('/private/SECRET', { async writeRows() {} }).catch(
|
||||
(error: unknown) => error
|
||||
);
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect((error as Error).message).not.toMatch(/SECRET|\/private/);
|
||||
expect((error as Error).cause).toBeUndefined();
|
||||
expect(processMock.spawn).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
@@ -2,11 +2,10 @@ import { getHledgerVersion, supportedHledgerVersion } from '../checks.js';
|
||||
import type { OverviewData } from '../overview/report.js';
|
||||
import { consumeHledger, readHledger } from '../process.js';
|
||||
import { ReportStreamReader } from '../report-stream.js';
|
||||
import { readPreparationMetadata } from './metadata.js';
|
||||
import { writePreparedAccountRows, type AccountRowWriter } from './account-stream.js';
|
||||
import {
|
||||
decodeDiscovery,
|
||||
decodePreparedPrefix,
|
||||
planDiscoveryCommands,
|
||||
planPreparedReportCommands,
|
||||
registersMarker,
|
||||
type PreparedAccountSummary
|
||||
@@ -24,22 +23,25 @@ export async function prepareReports(
|
||||
await getHledgerVersion((args) => readHledger(args, { signal: options.signal }));
|
||||
options.signal?.throwIfAborted();
|
||||
const inputFlags = ['--no-conf', '--lang=en', '--strict', '-f', journal];
|
||||
const discoveryText = await readHledger(planDiscoveryCommands(inputFlags), {
|
||||
signal: options.signal
|
||||
});
|
||||
const metadata = await readPreparationMetadata(inputFlags, { signal: options.signal });
|
||||
options.signal?.throwIfAborted();
|
||||
const { names, discovery } = decodeDiscovery(discoveryText);
|
||||
const { args, overviewPlan } = planPreparedReportCommands(inputFlags, discovery, names);
|
||||
const { args, overviewPlan } = planPreparedReportCommands(inputFlags, metadata);
|
||||
const { overview, accounts } = await consumeHledger(
|
||||
args,
|
||||
{ signal: options.signal },
|
||||
async (output, check) => {
|
||||
const reader = new ReportStreamReader(output, check);
|
||||
const { overview, accounts, metadata } = await reader.prefix(registersMarker, (prefix) =>
|
||||
decodePreparedPrefix(prefix, names, discovery, overviewPlan)
|
||||
const { overview, accounts } = await reader.prefix(registersMarker, (prefix) =>
|
||||
decodePreparedPrefix(prefix, metadata, overviewPlan)
|
||||
);
|
||||
for (const account of accounts) {
|
||||
await writePreparedAccountRows(reader, account, metadata, options.writeRows, check);
|
||||
await writePreparedAccountRows(
|
||||
reader,
|
||||
account,
|
||||
metadata.investment,
|
||||
options.writeRows,
|
||||
check
|
||||
);
|
||||
}
|
||||
await reader.eof();
|
||||
return { overview, accounts };
|
||||
|
||||
@@ -10,6 +10,19 @@ type Entry =
|
||||
| { kind: 'original'; transaction: NativeTransaction }
|
||||
| { kind: 'joined'; transaction: PreparedTransaction };
|
||||
|
||||
export function decodeNativeTransaction(json: string): NativeTransaction {
|
||||
const value = parseExactJson(json) as NativeTransaction;
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!Array.isArray(value.tpostings)
|
||||
)
|
||||
throw new Error('Invalid native print transaction.');
|
||||
requireTransactionIndex(value.tindex);
|
||||
return value;
|
||||
}
|
||||
|
||||
/** One account only. Never retain the wider case-insensitive print selection. */
|
||||
export class AccountTransactionExports {
|
||||
private readonly entries = new Map<number, Entry>();
|
||||
@@ -21,19 +34,6 @@ export class AccountTransactionExports {
|
||||
this.ambiguous = ambiguous;
|
||||
}
|
||||
|
||||
private decode(json: string): NativeTransaction {
|
||||
const value = parseExactJson(json) as NativeTransaction;
|
||||
if (
|
||||
!value ||
|
||||
typeof value !== 'object' ||
|
||||
Array.isArray(value) ||
|
||||
!Array.isArray(value.tpostings)
|
||||
)
|
||||
throw new Error('Invalid native print transaction.');
|
||||
requireTransactionIndex(value.tindex);
|
||||
return value;
|
||||
}
|
||||
|
||||
private selected(transaction: NativeTransaction): boolean {
|
||||
return transaction.tpostings.some(({ paccount }) => {
|
||||
if (typeof paccount !== 'string') throw new Error('Invalid native posting account.');
|
||||
@@ -42,7 +42,7 @@ export class AccountTransactionExports {
|
||||
}
|
||||
|
||||
original(json: string): void {
|
||||
const transaction = this.decode(json);
|
||||
const transaction = decodeNativeTransaction(json);
|
||||
if (this.ambiguous || !this.selected(transaction)) return;
|
||||
if (this.entries.has(transaction.tindex)) throw new Error('Duplicate native original index.');
|
||||
// Keep the native original until its non-amount structure can be checked.
|
||||
@@ -50,7 +50,7 @@ export class AccountTransactionExports {
|
||||
}
|
||||
|
||||
transacted(json: string): void {
|
||||
const native = this.decode(json);
|
||||
const native = decodeNativeTransaction(json);
|
||||
if (this.ambiguous) return;
|
||||
const entry = this.entries.get(native.tindex);
|
||||
if (!entry) {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
export { decodeNativeTransaction } from './export.js';
|
||||
export { assertTransactionExportsMatch } from './transaction.js';
|
||||
export type { NativeTransaction } from './transaction.js';
|
||||
@@ -295,10 +295,10 @@ export function assertTransactionIdentity(
|
||||
throw new Error('Mismatched native transaction header.');
|
||||
}
|
||||
|
||||
export function prepareTransaction(
|
||||
function matchingTransactedPostings(
|
||||
transaction: NativeTransaction,
|
||||
transacted: NativeTransaction
|
||||
): PreparedTransaction {
|
||||
): (NativePosting | null)[] {
|
||||
const { tpostings: originals, ...originalHeader } = transaction;
|
||||
const { tpostings: valued, ...valuedHeader } = transacted;
|
||||
requireTransactionIndex(transaction.tindex);
|
||||
@@ -311,11 +311,11 @@ export function prepareTransaction(
|
||||
throw new Error('Mismatched native print exports.');
|
||||
}
|
||||
let next = 0;
|
||||
const postings = originals.map((posting) => {
|
||||
const matches = originals.map((posting) => {
|
||||
const omitted =
|
||||
posting.ptags.some(([key]) => key === '_conversion-posting') &&
|
||||
posting.pamount.every((amount) => amount.acost === null);
|
||||
if (omitted) return preparePosting(posting, null, transaction.tstatus);
|
||||
if (omitted) return null;
|
||||
const counterpart = valued[next++];
|
||||
if (!counterpart) throw new Error('Missing native transacted posting.');
|
||||
const { pamount: originalAmounts, ...metadata } = posting;
|
||||
@@ -325,9 +325,27 @@ export function prepareTransaction(
|
||||
if (!isDeepStrictEqual(metadata, valuedMetadata)) {
|
||||
throw new Error('Mismatched native posting metadata.');
|
||||
}
|
||||
return preparePosting(posting, counterpart, transaction.tstatus);
|
||||
return counterpart;
|
||||
});
|
||||
if (next !== valued.length) throw new Error('Extra native transacted posting.');
|
||||
return matches;
|
||||
}
|
||||
|
||||
export function assertTransactionExportsMatch(
|
||||
transaction: NativeTransaction,
|
||||
transacted: NativeTransaction
|
||||
): void {
|
||||
matchingTransactedPostings(transaction, transacted);
|
||||
}
|
||||
|
||||
export function prepareTransaction(
|
||||
transaction: NativeTransaction,
|
||||
transacted: NativeTransaction
|
||||
): PreparedTransaction {
|
||||
const matches = matchingTransactedPostings(transaction, transacted);
|
||||
const postings = matches.map((counterpart, index) =>
|
||||
preparePosting(transaction.tpostings[index], counterpart, transaction.tstatus)
|
||||
);
|
||||
return projectOriginalTransaction(transaction, postings);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user