From 84b26d29f94ca60a09de41489f004567919daed5 Mon Sep 17 00:00:00 2001 From: eleith Date: Sat, 7 Nov 2026 11:28:59 -0800 Subject: [PATCH 1/2] feat(analytics): add shared report document contracts --- .changeset/config.json | 2 +- packages/analytics/package.json | 34 +++++ packages/analytics/src/decode.test.ts | 102 +++++++++++++ packages/analytics/src/decode.ts | 24 +++ packages/analytics/src/encode.test.ts | 200 +++++++++++++++++++++++++ packages/analytics/src/encode.ts | 119 +++++++++++++++ packages/analytics/src/index.ts | 16 ++ packages/analytics/src/period.test.ts | 51 +++++++ packages/analytics/src/period.ts | 54 +++++++ packages/analytics/src/reports.test.ts | 37 +++++ packages/analytics/src/reports.ts | 123 +++++++++++++++ packages/analytics/src/validation.ts | 73 +++++++++ packages/analytics/tsconfig.build.json | 6 + packages/analytics/tsconfig.json | 11 ++ packages/analytics/vitest.config.ts | 13 ++ pnpm-lock.yaml | 22 +++ 16 files changed, 886 insertions(+), 1 deletion(-) create mode 100644 packages/analytics/package.json create mode 100644 packages/analytics/src/decode.test.ts create mode 100644 packages/analytics/src/decode.ts create mode 100644 packages/analytics/src/encode.test.ts create mode 100644 packages/analytics/src/encode.ts create mode 100644 packages/analytics/src/index.ts create mode 100644 packages/analytics/src/period.test.ts create mode 100644 packages/analytics/src/period.ts create mode 100644 packages/analytics/src/reports.test.ts create mode 100644 packages/analytics/src/reports.ts create mode 100644 packages/analytics/src/validation.ts create mode 100644 packages/analytics/tsconfig.build.json create mode 100644 packages/analytics/tsconfig.json create mode 100644 packages/analytics/vitest.config.ts diff --git a/.changeset/config.json b/.changeset/config.json index dd88f3f..0666016 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -7,7 +7,7 @@ "access": "restricted", "baseBranch": "main", "updateInternalDependencies": "patch", - "ignore": ["@goto/config", "@goto/db"], + "ignore": ["@goto/analytics", "@goto/config", "@goto/db"], "privatePackages": { "version": true, "tag": true diff --git a/packages/analytics/package.json b/packages/analytics/package.json new file mode 100644 index 0000000..7a7da12 --- /dev/null +++ b/packages/analytics/package.json @@ -0,0 +1,34 @@ +{ + "name": "@goto/analytics", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": { + "development": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.build.json", + "check": "tsc --noEmit -p tsconfig.json", + "lint": "prettier --ignore-path ../../.prettierignore --check . && eslint .", + "format": "prettier --ignore-path ../../.prettierignore --write .", + "test": "vitest run", + "test:coverage": "vitest run --coverage" + }, + "dependencies": { + "@sinclair/typebox": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "@vitest/coverage-v8": "catalog:", + "prettier": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/analytics/src/decode.test.ts b/packages/analytics/src/decode.test.ts new file mode 100644 index 0000000..383d438 --- /dev/null +++ b/packages/analytics/src/decode.test.ts @@ -0,0 +1,102 @@ +import { expect, test } from 'vitest'; +import { parseVisitDetails, parseVisitSummary } from './decode.js'; +import { createVisitPeriod, visitDays } from './period.js'; +import { zeroVisitTotals, type OverviewSummary, type VisitDetails } from './reports.js'; + +const scope = { kind: 'overview' } as const; +const period = createVisitPeriod(3, Date.UTC(2026, 3, 1, 12)); +const summary: OverviewSummary = { + version: 1, + scope, + period, + totals: { validVisits: 2, invalidVisits: 1 }, + base: { validVisits: 1, invalidVisits: 0 }, + domains: [] +}; +const details: VisitDetails = { + version: 1, + scope, + period, + daily: visitDays(period).map((day, index) => ({ + ...day, + ...(index === 0 ? summary.totals : zeroVisitTotals()) + })), + browsers: [{ family: 'Firefox', ...summary.totals }], + operatingSystems: [{ family: 'Linux', ...summary.totals }] +}; + +test('readers decode their self-contained document independently', () => { + expect(parseVisitSummary(JSON.stringify(summary), scope)).toEqual(summary); + expect(parseVisitDetails(JSON.stringify(details), scope)).toEqual(details); +}); + +test('summary reader rejects unsupported versions, unsafe counts and wrong identities safely', () => { + for (const invalid of [ + { ...summary, version: 2 }, + { ...summary, totals: { validVisits: -1, invalidVisits: 0 } }, + { ...summary, totals: { validVisits: 0.5, invalidVisits: 0 } }, + { ...summary, totals: { validVisits: Number.MAX_SAFE_INTEGER + 1, invalidVisits: 0 } }, + { ...summary, unexpected: 'private' } + ]) { + expect(() => parseVisitSummary(JSON.stringify(invalid), scope)).toThrow( + 'Invalid analytics report' + ); + } + expect(() => + parseVisitSummary(JSON.stringify(summary), { kind: 'domain', domain: 'private' }) + ).toThrow('Invalid analytics report'); + expect(() => parseVisitSummary('{private', scope)).toThrow('Invalid analytics report'); +}); + +test('readers reject noncanonical, reversed and non-retention windows', () => { + for (const invalidPeriod of [ + { ...period, asOf: 'not-a-date' }, + { ...period, asOf: period.asOf.replace('.000Z', 'Z') }, + { ...period, from: '2026-02-30T12:00:00.000Z' }, + { ...period, from: period.asOf }, + { ...period, from: '2025-01-01T00:00:00.000Z' } + ]) { + expect(() => + parseVisitSummary(JSON.stringify({ ...summary, period: invalidPeriod }), scope) + ).toThrow('Invalid analytics report'); + expect(() => + parseVisitDetails(JSON.stringify({ ...details, period: invalidPeriod }), scope) + ).toThrow('Invalid analytics report'); + } +}); + +test('details reader checks daily instants, window safety, counts and scope', () => { + const first = details.daily[0]; + if (!first) throw new Error('Expected a daily fixture'); + for (const point of [ + { ...first, start: 'not-a-date' }, + { ...first, end: first.end.replace('.000Z', 'Z') }, + { ...first, start: '2026-02-30T00:00:00.000Z' }, + { ...first, start: first.end }, + { ...first, start: period.asOf, end: '2026-04-02T00:00:00.000Z' }, + { ...first, invalidVisits: -1 } + ]) { + expect(() => parseVisitDetails(JSON.stringify({ ...details, daily: [point] }), scope)).toThrow( + 'Invalid analytics report' + ); + } + expect(() => parseVisitDetails(JSON.stringify({ ...details, version: 2 }), scope)).toThrow( + 'Invalid analytics report' + ); + expect(() => + parseVisitDetails(JSON.stringify(details), { kind: 'domain', domain: 'private' }) + ).toThrow('Invalid analytics report'); + expect(() => parseVisitDetails('{private', scope)).toThrow('Invalid analytics report'); +}); + +test('reader prevents a base subset from exceeding totals used by page subtraction', () => { + expect(() => + parseVisitSummary( + JSON.stringify({ + ...summary, + base: { validVisits: 3, invalidVisits: 0 } + }), + scope + ) + ).toThrow('Invalid analytics report'); +}); diff --git a/packages/analytics/src/decode.ts b/packages/analytics/src/decode.ts new file mode 100644 index 0000000..1b058df --- /dev/null +++ b/packages/analytics/src/decode.ts @@ -0,0 +1,24 @@ +import type { ReportScope, VisitDetails, VisitSummary } from './reports.js'; +import { checkDetails, checkSummary, invalidReport, sameScope } from './validation.js'; + +export function parseVisitSummary(json: string, expectedScope: ReportScope): VisitSummary { + const document = parseDocument(json); + const summary = checkSummary(document); + if (!sameScope(summary.scope, expectedScope)) throw invalidReport(); + return summary; +} + +export function parseVisitDetails(json: string, expectedScope: ReportScope): VisitDetails { + const document = parseDocument(json); + const details = checkDetails(document); + if (!sameScope(details.scope, expectedScope)) throw invalidReport(); + return details; +} + +function parseDocument(json: string): unknown { + try { + return JSON.parse(json); + } catch { + throw invalidReport(); + } +} diff --git a/packages/analytics/src/encode.test.ts b/packages/analytics/src/encode.test.ts new file mode 100644 index 0000000..bd657fe --- /dev/null +++ b/packages/analytics/src/encode.test.ts @@ -0,0 +1,200 @@ +import { expect, test, vi } from 'vitest'; +import { parseVisitDetails, parseVisitSummary } from './decode.js'; +import { encodeVisitReports } from './encode.js'; +import { createVisitPeriod, visitDays } from './period.js'; +import { + zeroVisitTotals, + type PreparedVisitReport, + type ReportScope, + type VisitDetails, + type VisitSummary, + type VisitTotals +} from './reports.js'; + +const overview = { kind: 'overview' } as const; +const domain = { kind: 'domain', domain: 'go.example.test' } as const; +const root = { kind: 'link', domain: domain.domain, path: '/' } as const; +const asOf = Date.UTC(2026, 3, 1, 12); + +function report( + scope: ReportScope = overview, + cutoff = asOf, + totals: VisitTotals = { validVisits: 2, invalidVisits: 1 } +): PreparedVisitReport { + const period = createVisitPeriod(3, cutoff); + let summary: VisitSummary; + if (scope.kind === 'overview') { + summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), domains: [] }; + } else if (scope.kind === 'domain') { + summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), links: [] }; + } else { + summary = { version: 1, scope, period, totals }; + } + const populated = totals.validVisits > 0 || totals.invalidVisits > 0; + return { + summary, + details: { + version: 1, + scope, + period, + daily: visitDays(period).map((day, index) => ({ + ...day, + ...(index === 0 ? totals : zeroVisitTotals()) + })), + browsers: populated ? [{ family: 'Firefox', ...totals }] : [], + operatingSystems: populated ? [{ family: 'Linux', ...totals }] : [] + } + }; +} + +test('encodes each summary variant and explicit root without changing document shapes', () => { + const reports = [report(overview), report(domain), report(root, asOf, zeroVisitTotals())]; + const rows = encodeVisitReports(reports); + + for (const [index, prepared] of reports.entries()) { + const row = rows[index]; + if (!row) throw new Error('Expected a serialized fixture'); + expect(row.scope).toEqual(prepared.summary.scope); + expect(parseVisitSummary(row.summaryJson, row.scope)).toEqual(prepared.summary); + expect(parseVisitDetails(row.detailsJson, row.scope)).toEqual(prepared.details); + } + expect(encodeVisitReports([])).toEqual([]); +}); + +test('encodes each document once without a stringify-to-parse validation loop', () => { + const reports = [report(overview), report(domain)]; + const stringify = vi.spyOn(JSON, 'stringify'); + const parse = vi.spyOn(JSON, 'parse'); + try { + encodeVisitReports(reports); + expect(stringify).toHaveBeenCalledTimes(4); + expect(parse).not.toHaveBeenCalled(); + } finally { + stringify.mockRestore(); + parse.mockRestore(); + } +}); + +test('rejects incompatible pair identities, periods, totals and document versions', () => { + const prepared = report(); + for (const invalid of [ + { ...prepared, summary: { ...prepared.summary, totals: { validVisits: 3, invalidVisits: 1 } } }, + { ...prepared, details: { ...prepared.details, scope: root } }, + { + ...prepared, + details: { ...prepared.details, period: createVisitPeriod(3, asOf + 86_400_000) } + } + ]) { + expect(() => encodeVisitReports([invalid])).toThrow('Invalid analytics report'); + } + const wrongVersion = report(); + Object.assign(wrongVersion.details, { version: 2 }); + expect(() => encodeVisitReports([wrongVersion])).toThrow('Invalid analytics report'); +}); + +test('batch checks duplicate scope identities and shared periods before serialization', () => { + const stringify = vi.spyOn(JSON, 'stringify'); + try { + for (const scope of [overview, domain, root]) { + expect(() => encodeVisitReports([report(scope), report(scope)])).toThrow( + 'Invalid analytics report' + ); + } + expect(() => encodeVisitReports([report(overview), report(domain, asOf + 86_400_000)])).toThrow( + 'Invalid analytics report' + ); + expect(stringify).not.toHaveBeenCalled(); + } finally { + stringify.mockRestore(); + } +}); + +test('producer requires gap-free UTC days and correct partial flags including zero points', () => { + const prepared = report(); + const daily = prepared.details.daily; + const first = daily[0]; + if (!first) throw new Error('Expected a daily fixture'); + const invalidDaily: VisitDetails['daily'][] = [ + daily.slice(1), + [...daily].reverse(), + [{ ...first, start: prepared.details.period.from }, ...daily.slice(1)], + [{ ...first, end: prepared.details.period.asOf }, ...daily.slice(1)], + [{ ...first, partial: false }, ...daily.slice(1)] + ]; + for (const points of invalidDaily) { + expect(() => + encodeVisitReports([ + { + ...prepared, + details: { ...prepared.details, daily: points } + } + ]) + ).toThrow('Invalid analytics report'); + } +}); + +test('producer checks family sums and duplicate labels against the daily totals', () => { + const prepared = report(); + for (const details of [ + { ...prepared.details, browsers: [] }, + { + ...prepared.details, + operatingSystems: [{ family: 'Linux', validVisits: 3, invalidVisits: 0 }] + }, + { + ...prepared.details, + browsers: [ + { family: 'Firefox', validVisits: 1, invalidVisits: 0 }, + { family: 'Firefox', validVisits: 1, invalidVisits: 1 } + ] + } + ]) { + expect(() => encodeVisitReports([{ ...prepared, details }])).toThrow( + 'Invalid analytics report' + ); + } +}); + +test('producer rejects base or compact child subsets above totals and duplicate children', () => { + const prepared = report(); + for (const summary of [ + { ...prepared.summary, base: { validVisits: 3, invalidVisits: 0 } }, + { + ...prepared.summary, + domains: [ + { domain: domain.domain, totals: zeroVisitTotals() }, + { domain: domain.domain, totals: zeroVisitTotals() } + ] + }, + { + ...prepared.summary, + domains: [{ domain: domain.domain, totals: { validVisits: 5, invalidVisits: 0 } }] + } + ]) { + expect(() => encodeVisitReports([{ ...prepared, summary }])).toThrow( + 'Invalid analytics report' + ); + } + const domainReport = report(domain); + const duplicateLinks = { + ...domainReport.summary, + links: [ + { path: '/', totals: zeroVisitTotals() }, + { path: '/', totals: zeroVisitTotals() } + ] + }; + expect(() => encodeVisitReports([{ ...domainReport, summary: duplicateLinks }])).toThrow( + 'Invalid analytics report' + ); +}); + +test('producer rejects unsafe summed counts even when each input count is safe', () => { + const prepared = report(overview, asOf, { + validVisits: Number.MAX_SAFE_INTEGER, + invalidVisits: 0 + }); + const secondDay = prepared.details.daily[1]; + if (!secondDay) throw new Error('Expected a second daily fixture'); + secondDay.validVisits = 1; + expect(() => encodeVisitReports([prepared])).toThrow('Invalid analytics report'); +}); diff --git a/packages/analytics/src/encode.ts b/packages/analytics/src/encode.ts new file mode 100644 index 0000000..4f9ba12 --- /dev/null +++ b/packages/analytics/src/encode.ts @@ -0,0 +1,119 @@ +import { visitDays } from './period.js'; +import { + zeroVisitTotals, + type PreparedVisitReport, + type SerializedVisitReport, + type VisitDetails, + type VisitSummary, + type VisitTotals +} from './reports.js'; +import { + checkDetails, + checkSummary, + equalTotals, + invalidReport, + samePeriod, + sameScope, + withinTotals +} from './validation.js'; + +export function encodeVisitReports(reports: PreparedVisitReport[]): SerializedVisitReport[] { + const first = reports[0]; + if (!first) return []; + let overview = false; + const domains = new Set(); + const links = new Map>(); + for (const report of reports) { + const summary = checkSummary(report.summary); + const details = checkDetails(report.details); + if ( + !sameScope(summary.scope, details.scope) || + !samePeriod(summary.period, details.period) || + !samePeriod(summary.period, first.summary.period) + ) { + throw invalidReport(); + } + const scope = summary.scope; + if (scope.kind === 'overview') { + if (overview) throw invalidReport(); + overview = true; + } else if (scope.kind === 'domain') { + if (domains.has(scope.domain)) throw invalidReport(); + domains.add(scope.domain); + } else { + const paths = links.get(scope.domain) ?? new Set(); + if (paths.has(scope.path)) throw invalidReport(); + paths.add(scope.path); + links.set(scope.domain, paths); + } + } + + const days = visitDays(first.summary.period); + for (const { summary, details } of reports) { + checkDailyLayout(details.daily, days); + const totals = sumTotals(details.daily); + if (!equalTotals(summary.totals, totals)) throw invalidReport(); + checkFamilies(details.browsers, totals); + checkFamilies(details.operatingSystems, totals); + checkChildren(summary); + } + + return reports.map(({ summary, details }) => ({ + scope: summary.scope, + summaryJson: JSON.stringify(summary), + detailsJson: JSON.stringify(details) + })); +} + +function checkDailyLayout(daily: VisitDetails['daily'], days: ReturnType): void { + if (daily.length !== days.length) throw invalidReport(); + for (const [index, day] of days.entries()) { + const point = daily[index]; + if ( + !point || + point.start !== day.start || + point.end !== day.end || + point.partial !== day.partial + ) { + throw invalidReport(); + } + } +} + +function checkFamilies(rows: VisitDetails['browsers'], totals: VisitTotals): void { + const families = new Set(rows.map(({ family }) => family)); + const combined = sumTotals(rows); + if (families.size !== rows.length || !equalTotals(combined, totals)) { + throw invalidReport(); + } +} + +function checkChildren(summary: VisitSummary): void { + if ('domains' in summary) { + if (new Set(summary.domains.map(({ domain }) => domain)).size !== summary.domains.length) { + throw invalidReport(); + } + const childTotals = summary.domains.map(({ totals }) => totals); + const combined = sumTotals(childTotals); + if (!withinTotals(combined, summary.totals)) throw invalidReport(); + } else if ('links' in summary) { + if (new Set(summary.links.map(({ path }) => path)).size !== summary.links.length) { + throw invalidReport(); + } + const childTotals = summary.links.map(({ totals }) => totals); + const combined = sumTotals(childTotals); + if (!withinTotals(combined, summary.totals)) throw invalidReport(); + } +} + +function sumTotals(rows: VisitTotals[]): VisitTotals { + const totals = zeroVisitTotals(); + for (const row of rows) { + totals.validVisits += row.validVisits; + totals.invalidVisits += row.invalidVisits; + } + if (!Number.isSafeInteger(totals.validVisits) || !Number.isSafeInteger(totals.invalidVisits)) { + throw invalidReport(); + } + return totals; +} diff --git a/packages/analytics/src/index.ts b/packages/analytics/src/index.ts new file mode 100644 index 0000000..f86bb4e --- /dev/null +++ b/packages/analytics/src/index.ts @@ -0,0 +1,16 @@ +export { createVisitPeriod, visitDays } from './period.js'; +export { zeroVisitTotals } from './reports.js'; +export { encodeVisitReports } from './encode.js'; +export { parseVisitSummary, parseVisitDetails } from './decode.js'; +export type { + ReportScope, + ReportPeriod, + VisitTotals, + VisitSummary, + OverviewSummary, + DomainSummary, + LinkSummary, + VisitDetails, + PreparedVisitReport, + SerializedVisitReport +} from './reports.js'; diff --git a/packages/analytics/src/period.test.ts b/packages/analytics/src/period.test.ts new file mode 100644 index 0000000..31302bf --- /dev/null +++ b/packages/analytics/src/period.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from 'vitest'; +import { createVisitPeriod, visitDays } from './period.js'; + +test('period subtraction clamps UTC month ends and preserves time of day', () => { + expect(createVisitPeriod(3, Date.UTC(2024, 4, 31, 12, 34, 56, 789))).toEqual({ + from: '2024-02-29T12:34:56.789Z', + asOf: '2024-05-31T12:34:56.789Z', + timezone: 'UTC', + retentionMonths: 3 + }); + expect(createVisitPeriod(9, Date.UTC(2026, 0, 31)).from).toBe('2025-04-30T00:00:00.000Z'); + expect(createVisitPeriod(12, Date.UTC(2024, 1, 29)).from).toBe('2023-02-28T00:00:00.000Z'); +}); + +test('daily points cover the full window in UTC with partial edge days', () => { + const period = createVisitPeriod(3, Date.UTC(2026, 4, 31, 12)); + const days = visitDays(period); + + expect(days[0]).toEqual({ + start: '2026-02-28T00:00:00.000Z', + end: '2026-03-01T00:00:00.000Z', + partial: true + }); + expect(days.at(-1)).toEqual({ + start: '2026-05-31T00:00:00.000Z', + end: '2026-06-01T00:00:00.000Z', + partial: true + }); + expect(days.slice(1, -1).every((day) => !day.partial)).toBe(true); + for (let index = 1; index < days.length; index++) { + expect(days[index]?.start).toBe(days[index - 1]?.end); + } +}); + +test('midnight cutoff omits the following day and leaves full days unmarked', () => { + const days = visitDays(createVisitPeriod(3, Date.UTC(2026, 3, 1))); + + expect(days).toHaveLength(90); + expect(days[0]?.start).toBe('2026-01-01T00:00:00.000Z'); + expect(days.at(-1)?.end).toBe('2026-04-01T00:00:00.000Z'); + expect(days.every((day) => !day.partial)).toBe(true); +}); + +test('period helpers reject invalid instants instead of normalizing dates silently', () => { + expect(() => createVisitPeriod(3, Number.NaN)).toThrow('Invalid analytics period'); + const period = createVisitPeriod(3, Date.UTC(2026, 3, 1)); + expect(() => visitDays({ ...period, from: '2026-02-30T00:00:00.000Z' })).toThrow( + 'Invalid analytics period' + ); + expect(() => visitDays({ ...period, from: period.asOf })).toThrow('Invalid analytics period'); +}); diff --git a/packages/analytics/src/period.ts b/packages/analytics/src/period.ts new file mode 100644 index 0000000..de4b8a6 --- /dev/null +++ b/packages/analytics/src/period.ts @@ -0,0 +1,54 @@ +import type { ReportPeriod } from './reports.js'; + +export function isCanonicalInstant(value: string): boolean { + const time = Date.parse(value); + return Number.isFinite(time) && new Date(time).toISOString() === value; +} + +export function createVisitPeriod(months: 3 | 9 | 12, asOf: number): ReportPeriod { + if (![3, 9, 12].includes(months) || !Number.isFinite(asOf)) { + throw new Error('Invalid analytics period'); + } + const end = new Date(asOf); + const start = new Date(asOf); + const day = start.getUTCDate(); + start.setUTCDate(1); + start.setUTCMonth(start.getUTCMonth() - months); + const lastDay = new Date(start); + lastDay.setUTCMonth(lastDay.getUTCMonth() + 1); + lastDay.setUTCDate(0); + start.setUTCDate(Math.min(day, lastDay.getUTCDate())); + if (!Number.isFinite(start.getTime()) || !Number.isFinite(end.getTime())) { + throw new Error('Invalid analytics period'); + } + return { + from: start.toISOString(), + asOf: end.toISOString(), + timezone: 'UTC', + retentionMonths: months + }; +} + +export function visitDays( + period: ReportPeriod +): Array<{ start: string; end: string; partial: boolean }> { + const from = Date.parse(period.from); + const asOf = Date.parse(period.asOf); + if (!isCanonicalInstant(period.from) || !isCanonicalInstant(period.asOf) || from >= asOf) { + throw new Error('Invalid analytics period'); + } + const date = new Date(from); + date.setUTCHours(0, 0, 0, 0); + const days = []; + while (date.getTime() < asOf) { + const start = date.toISOString(); + const startTime = date.getTime(); + date.setUTCDate(date.getUTCDate() + 1); + days.push({ + start, + end: date.toISOString(), + partial: startTime < from || date.getTime() > asOf + }); + } + return days; +} diff --git a/packages/analytics/src/reports.test.ts b/packages/analytics/src/reports.test.ts new file mode 100644 index 0000000..6d7d189 --- /dev/null +++ b/packages/analytics/src/reports.test.ts @@ -0,0 +1,37 @@ +import { Value } from '@sinclair/typebox/value'; +import { expect, test } from 'vitest'; +import { createVisitPeriod } from './period.js'; +import { + ReportScopeSchema, + VisitSummarySchema, + zeroVisitTotals, + type LinkSummary +} from './reports.js'; + +test('zero totals are independent objects with no redundant total or root count', () => { + const first = zeroVisitTotals(); + const second = zeroVisitTotals(); + first.validVisits = 1; + + expect(second).toEqual({ validVisits: 0, invalidVisits: 0 }); + expect(first).not.toBe(second); +}); + +test('canonical schemas distinguish scopes and summary variants', () => { + const summary: LinkSummary = { + version: 1, + scope: { kind: 'link', domain: 'go.example.test', path: '/' }, + period: createVisitPeriod(3, Date.UTC(2026, 2, 1)), + totals: zeroVisitTotals() + }; + + expect(Value.Check(VisitSummarySchema, summary)).toBe(true); + expect(Value.Check(VisitSummarySchema, { ...summary, base: zeroVisitTotals() })).toBe(false); + expect(Value.Check(ReportScopeSchema, { kind: 'overview', domain: null })).toBe(false); + expect( + Value.Check(ReportScopeSchema, { kind: 'domain', domain: 'go.example.test', path: '/' }) + ).toBe(false); + expect( + Value.Check(ReportScopeSchema, { kind: 'link', domain: 'go.example.test', path: '' }) + ).toBe(false); +}); diff --git a/packages/analytics/src/reports.ts b/packages/analytics/src/reports.ts new file mode 100644 index 0000000..75fb328 --- /dev/null +++ b/packages/analytics/src/reports.ts @@ -0,0 +1,123 @@ +import { FormatRegistry, Type, type Static } from '@sinclair/typebox'; +import { isCanonicalInstant } from './period.js'; + +const exact = { additionalProperties: false }; +const count = Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }); +const domain = Type.String({ minLength: 1 }); +const path = Type.String({ pattern: '^/' }); + +FormatRegistry.Set('goto-analytics-utc-instant', isCanonicalInstant); +const instant = Type.String({ format: 'goto-analytics-utc-instant' }); + +export const ReportScopeSchema = Type.Union([ + Type.Object({ kind: Type.Literal('overview') }, exact), + Type.Object({ kind: Type.Literal('domain'), domain }, exact), + Type.Object({ kind: Type.Literal('link'), domain, path }, exact) +]); +export type ReportScope = Static; + +export const VisitTotalsSchema = Type.Object({ validVisits: count, invalidVisits: count }, exact); +export type VisitTotals = Static; + +export const ReportPeriodSchema = Type.Object( + { + from: instant, + asOf: instant, + timezone: Type.Literal('UTC'), + retentionMonths: Type.Union([Type.Literal(3), Type.Literal(9), Type.Literal(12)]) + }, + exact +); +export type ReportPeriod = Static; + +const headers = { + version: Type.Literal(1), + period: ReportPeriodSchema, + totals: VisitTotalsSchema +}; + +export const OverviewSummarySchema = Type.Object( + { + ...headers, + scope: Type.Object({ kind: Type.Literal('overview') }, exact), + base: VisitTotalsSchema, + domains: Type.Array(Type.Object({ domain, totals: VisitTotalsSchema }, exact)) + }, + exact +); +export type OverviewSummary = Static; + +export const DomainSummarySchema = Type.Object( + { + ...headers, + scope: Type.Object({ kind: Type.Literal('domain'), domain }, exact), + base: VisitTotalsSchema, + links: Type.Array(Type.Object({ path, totals: VisitTotalsSchema }, exact)) + }, + exact +); +export type DomainSummary = Static; + +export const LinkSummarySchema = Type.Object( + { + ...headers, + scope: Type.Object({ kind: Type.Literal('link'), domain, path }, exact) + }, + exact +); +export type LinkSummary = Static; + +export const VisitSummarySchema = Type.Union([ + OverviewSummarySchema, + DomainSummarySchema, + LinkSummarySchema +]); +export type VisitSummary = Static; + +const familyCounts = Type.Object( + { + family: Type.String({ minLength: 1 }), + validVisits: count, + invalidVisits: count + }, + exact +); + +export const VisitDetailsSchema = Type.Object( + { + version: Type.Literal(1), + scope: ReportScopeSchema, + period: ReportPeriodSchema, + daily: Type.Array( + Type.Object( + { + start: instant, + end: instant, + partial: Type.Boolean(), + validVisits: count, + invalidVisits: count + }, + exact + ) + ), + browsers: Type.Array(familyCounts), + operatingSystems: Type.Array(familyCounts) + }, + exact +); +export type VisitDetails = Static; + +export interface PreparedVisitReport { + summary: VisitSummary; + details: VisitDetails; +} + +export interface SerializedVisitReport { + scope: ReportScope; + summaryJson: string; + detailsJson: string; +} + +export function zeroVisitTotals(): VisitTotals { + return { validVisits: 0, invalidVisits: 0 }; +} diff --git a/packages/analytics/src/validation.ts b/packages/analytics/src/validation.ts new file mode 100644 index 0000000..f786483 --- /dev/null +++ b/packages/analytics/src/validation.ts @@ -0,0 +1,73 @@ +import { Value } from '@sinclair/typebox/value'; +import { createVisitPeriod } from './period.js'; +import { + VisitDetailsSchema, + VisitSummarySchema, + type ReportPeriod, + type ReportScope, + type VisitDetails, + type VisitSummary, + type VisitTotals +} from './reports.js'; + +export function checkSummary(value: unknown): VisitSummary { + if (!Value.Check(VisitSummarySchema, value)) throw invalidReport(); + checkPeriod(value.period); + if ('base' in value && !withinTotals(value.base, value.totals)) throw invalidReport(); + return value; +} + +export function checkDetails(value: unknown): VisitDetails { + if (!Value.Check(VisitDetailsSchema, value)) throw invalidReport(); + checkPeriod(value.period); + const from = Date.parse(value.period.from); + const asOf = Date.parse(value.period.asOf); + const dayMilliseconds = 86_400_000; + const earliestStart = Math.floor(from / dayMilliseconds) * dayMilliseconds; + const latestEnd = Math.ceil(asOf / dayMilliseconds) * dayMilliseconds; + for (const point of value.daily) { + const start = Date.parse(point.start); + const end = Date.parse(point.end); + if (start < earliestStart || end > latestEnd || start >= end || start >= asOf || end <= from) { + throw invalidReport(); + } + } + return value; +} + +function checkPeriod(period: ReportPeriod): void { + let expected: ReportPeriod; + try { + expected = createVisitPeriod(period.retentionMonths, Date.parse(period.asOf)); + } catch { + throw invalidReport(); + } + if (!samePeriod(period, expected)) throw invalidReport(); +} + +export function samePeriod(left: ReportPeriod, right: ReportPeriod): boolean { + return ( + left.from === right.from && + left.asOf === right.asOf && + left.timezone === right.timezone && + left.retentionMonths === right.retentionMonths + ); +} + +export function sameScope(left: ReportScope, right: ReportScope): boolean { + if (left.kind === 'overview') return right.kind === 'overview'; + if (left.kind === 'domain') return right.kind === 'domain' && left.domain === right.domain; + return right.kind === 'link' && left.domain === right.domain && left.path === right.path; +} + +export function equalTotals(left: VisitTotals, right: VisitTotals): boolean { + return left.validVisits === right.validVisits && left.invalidVisits === right.invalidVisits; +} + +export function withinTotals(subset: VisitTotals, totals: VisitTotals): boolean { + return subset.validVisits <= totals.validVisits && subset.invalidVisits <= totals.invalidVisits; +} + +export function invalidReport(): Error { + return new Error('Invalid analytics report'); +} diff --git a/packages/analytics/tsconfig.build.json b/packages/analytics/tsconfig.build.json new file mode 100644 index 0000000..4609f66 --- /dev/null +++ b/packages/analytics/tsconfig.build.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { "rootDir": "src" }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "src/**/*.test.ts"] +} diff --git a/packages/analytics/tsconfig.json b/packages/analytics/tsconfig.json new file mode 100644 index 0000000..347fd04 --- /dev/null +++ b/packages/analytics/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "rootDir": ".", + "outDir": "dist", + "types": ["node"] + }, + "include": ["src/**/*.ts"], + "exclude": ["dist"] +} diff --git a/packages/analytics/vitest.config.ts b/packages/analytics/vitest.config.ts new file mode 100644 index 0000000..0e42110 --- /dev/null +++ b/packages/analytics/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + environment: 'node', + coverage: { + provider: 'v8', + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts'] + } + } +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc12213..ce76dfd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -244,6 +244,28 @@ importers: specifier: 'catalog:' version: 4.1.7(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.2)(yaml@2.9.1)) + packages/analytics: + dependencies: + '@sinclair/typebox': + specifier: 'catalog:' + version: 0.34.49 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.12.4 + '@vitest/coverage-v8': + specifier: 'catalog:' + version: 4.1.7(vitest@4.1.7) + prettier: + specifier: 'catalog:' + version: 3.8.3 + typescript: + specifier: 'catalog:' + version: 6.0.3 + vitest: + specifier: 'catalog:' + version: 4.1.7(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.2)(yaml@2.9.1)) + packages/config: dependencies: '@sinclair/typebox': -- 2.54.0 From ac0c9b48a5d65a05d4e7240f3d51042a18d009bb Mon Sep 17 00:00:00 2001 From: eleith Date: Sat, 7 Nov 2026 11:29:20 -0800 Subject: [PATCH 2/2] refactor(analytics): switch to daily JSON report storage --- .changeset/analytics-json-documents.md | 7 + .github/workflows/db.yml | 7 + .github/workflows/web.yml | 4 + .github/workflows/worker.yml | 7 + .woodpecker/db.yml | 5 + .woodpecker/web.yml | 2 + .woodpecker/worker.yml | 5 + apps/web/Dockerfile | 6 +- apps/web/docker-compose.dev.yml | 2 + apps/web/package.json | 1 + apps/web/src/lib/server/visit-reports.test.ts | 67 ++-- apps/web/src/lib/server/visit-reports.ts | 16 +- .../(admin)/(auth)/admin/+page.server.ts | 24 +- .../routes/(admin)/(auth)/admin/+page.svelte | 4 +- .../admin/domains/[domain]/+page.server.ts | 35 +- .../admin/domains/[domain]/+page.svelte | 6 +- .../[domain]/links/[...path]/+page.server.ts | 19 +- .../[domain]/links/[...path]/+page.svelte | 2 +- .../links/[...path]/page.server.test.ts | 80 +++-- .../domains/[domain]/page.server.test.ts | 93 +++-- .../(admin)/(auth)/admin/page.server.test.ts | 56 ++- apps/worker/package.json | 1 + .../src/services/build-visit-reports.test.ts | 252 ++++++++++++-- .../src/services/build-visit-reports.ts | 257 ++++++++------ .../services/prepare-visit-reports.test.ts | 93 +++-- .../src/services/prepare-visit-reports.ts | 12 +- .../src/services/report-buckets.test.ts | 33 -- apps/worker/src/services/report-buckets.ts | 36 -- .../src/services/report-schedule.test.ts | 11 +- apps/worker/src/services/report-schedule.ts | 4 +- apps/worker/src/services/retention.test.ts | 18 - apps/worker/src/services/retention.ts | 15 - apps/worker/src/services/startup.test.ts | 13 +- packages/db/package.json | 1 + .../src/migrations/0007_analytics_reports.ts | 41 +++ packages/db/src/migrations/index.ts | 4 +- packages/db/src/storage.test.ts | 28 +- packages/db/src/types.ts | 12 +- packages/db/src/visit-reports/index.ts | 17 +- packages/db/src/visit-reports/queries.test.ts | 255 +++++++++++--- packages/db/src/visit-reports/queries.ts | 57 ++-- packages/db/src/visit-reports/storage.test.ts | 322 +++++++++++------- packages/db/src/visit-reports/storage.ts | 101 +++--- packages/db/src/visit-reports/types.test.ts | 20 -- packages/db/src/visit-reports/types.ts | 77 +---- pnpm-lock.yaml | 9 + 46 files changed, 1311 insertions(+), 826 deletions(-) create mode 100644 .changeset/analytics-json-documents.md delete mode 100644 apps/worker/src/services/report-buckets.test.ts delete mode 100644 apps/worker/src/services/report-buckets.ts delete mode 100644 apps/worker/src/services/retention.test.ts delete mode 100644 apps/worker/src/services/retention.ts create mode 100644 packages/db/src/migrations/0007_analytics_reports.ts delete mode 100644 packages/db/src/visit-reports/types.test.ts diff --git a/.changeset/analytics-json-documents.md b/.changeset/analytics-json-documents.md new file mode 100644 index 0000000..18e1244 --- /dev/null +++ b/.changeset/analytics-json-documents.md @@ -0,0 +1,7 @@ +--- +'@goto/web': patch +'@goto/worker': patch +'@goto/cli': patch +--- + +Replace opaque analytics cache entries with versioned summary and daily-detail JSON documents identified by report scope. Separate shared analytics contracts and encoding from SQLite persistence. Keep admin reads summary-only, remove stored weekly/monthly series and shared empty/period fallbacks, and rebuild derived reports from retained visits after migration. Existing analytics collection, retention and admin behavior are unchanged. diff --git a/.github/workflows/db.yml b/.github/workflows/db.yml index 95cf9f3..f509879 100644 --- a/.github/workflows/db.yml +++ b/.github/workflows/db.yml @@ -9,6 +9,7 @@ on: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/analytics/** - packages/config/** - scripts/** - tools/** @@ -20,6 +21,7 @@ on: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/analytics/** - packages/config/** - scripts/** - tools/** @@ -48,6 +50,11 @@ jobs: run: pnpm install --frozen-lockfile - name: Build DB run: pnpm --filter '@goto/db...' build + - name: Check analytics dependency + run: | + pnpm --filter @goto/analytics check + pnpm --filter @goto/analytics lint + pnpm --filter @goto/analytics test:coverage - name: Check types run: pnpm --filter @goto/db check - name: Lint DB diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 5e624b4..14f6b95 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -55,6 +55,10 @@ jobs: run: pnpm --filter @goto/web check - name: Lint repository run: pnpm lint + - name: Check analytics dependency + run: | + pnpm --filter @goto/analytics check + pnpm --filter @goto/analytics test:coverage - name: Unit tests run: pnpm --filter @goto/web test:coverage diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml index 364bbb2..842e460 100644 --- a/.github/workflows/worker.yml +++ b/.github/workflows/worker.yml @@ -12,6 +12,7 @@ on: - apps/worker/** - packages/config/** - packages/db/** + - packages/analytics/** - tools/** pull_request: branches: [main] @@ -23,6 +24,7 @@ on: - apps/worker/** - packages/config/** - packages/db/** + - packages/analytics/** - tools/** permissions: @@ -55,6 +57,11 @@ jobs: run: pnpm --filter @goto/worker check - name: Lint worker run: pnpm --filter @goto/worker lint + - name: Check analytics dependency + run: | + pnpm --filter @goto/analytics check + pnpm --filter @goto/analytics lint + pnpm --filter @goto/analytics test:coverage - name: Check database dependency run: | pnpm --filter @goto/db check diff --git a/.woodpecker/db.yml b/.woodpecker/db.yml index d338e62..a1ee76d 100644 --- a/.woodpecker/db.yml +++ b/.woodpecker/db.yml @@ -12,6 +12,7 @@ when: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/analytics/** - packages/config/** - scripts/** - tools/** @@ -24,6 +25,7 @@ when: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/analytics/** - packages/config/** - scripts/** - tools/** @@ -56,6 +58,8 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics check + - pnpm --filter @goto/analytics lint - pnpm --filter @goto/db check - pnpm --filter @goto/db lint - pnpm exec prettier --check scripts @@ -68,5 +72,6 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics test:coverage - pnpm --filter @goto/db test:coverage - pnpm test:scripts diff --git a/.woodpecker/web.yml b/.woodpecker/web.yml index c87a6b4..4dee6a8 100644 --- a/.woodpecker/web.yml +++ b/.woodpecker/web.yml @@ -50,6 +50,7 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics check - pnpm --filter @goto/web check - pnpm lint @@ -60,6 +61,7 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics test:coverage - pnpm --filter @goto/web test:coverage e2e: diff --git a/.woodpecker/worker.yml b/.woodpecker/worker.yml index 8fb138f..c82fcd9 100644 --- a/.woodpecker/worker.yml +++ b/.woodpecker/worker.yml @@ -14,6 +14,7 @@ when: - apps/worker/** - packages/config/** - packages/db/** + - packages/analytics/** - tools/** - event: pull_request path: @@ -25,6 +26,7 @@ when: - apps/worker/** - packages/config/** - packages/db/** + - packages/analytics/** - tools/** - event: tag ref: 'refs/tags/@goto/worker@*' @@ -50,6 +52,8 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics check + - pnpm --filter @goto/analytics lint - pnpm --filter @goto/db check - pnpm --filter @goto/db lint - pnpm --filter @goto/worker check @@ -62,6 +66,7 @@ steps: commands: - npm install --global corepack@0.36.0 - corepack enable + - pnpm --filter @goto/analytics test:coverage - pnpm --filter @goto/db test:coverage - pnpm --filter @goto/worker test:coverage - pnpm --filter @goto/worker test:smoke diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 48131f3..3ef7afc 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -21,16 +21,16 @@ RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm fetch --frozen- COPY . . RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm install --offline --frozen-lockfile # Only generated output and caches are writable; source binds stay read-only. -RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/db/dist /m/apps/web/.svelte-kit \ +RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/analytics/dist /m/packages/db/dist /m/apps/web/.svelte-kit \ '/m/apps/web/node_modules/$app' \ /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp \ - && chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/db/dist \ + && chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/analytics/dist /m/packages/db/dist \ /m/apps/web/.svelte-kit '/m/apps/web/node_modules/$app' \ /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp USER ${LOCAL_UID}:${LOCAL_GID} WORKDIR /m/apps/web EXPOSE 5173 -CMD ["sh", "-c", "pnpm -r --filter @goto/config --filter @goto/db build && exec node_modules/.bin/vite --host 0.0.0.0 --port 5173 --strictPort"] +CMD ["sh", "-c", "pnpm -r --filter @goto/config --filter @goto/db... build && exec node_modules/.bin/vite --host 0.0.0.0 --port 5173 --strictPort"] FROM base AS builder diff --git a/apps/web/docker-compose.dev.yml b/apps/web/docker-compose.dev.yml index 6111137..11bf44a 100644 --- a/apps/web/docker-compose.dev.yml +++ b/apps/web/docker-compose.dev.yml @@ -28,6 +28,7 @@ services: - ./tsconfig.json:/m/apps/web/tsconfig.json:ro - ../../packages/config/src:/m/packages/config/src:ro - ../../packages/db/src:/m/packages/db/src:ro + - ../../packages/analytics/src:/m/packages/analytics/src:ro restart: unless-stopped healthcheck: test: @@ -64,6 +65,7 @@ services: - ../worker/src:/m/apps/worker/src:ro - ../../packages/config/src:/m/packages/config/src:ro - ../../packages/db/src:/m/packages/db/src:ro + - ../../packages/analytics/src:/m/packages/analytics/src:ro restart: unless-stopped healthcheck: test: diff --git a/apps/web/package.json b/apps/web/package.json index 7ef8ae0..4c59310 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -24,6 +24,7 @@ "test:e2e": "pnpm --filter @goto/worker... build && playwright test" }, "dependencies": { + "@goto/analytics": "workspace:*", "@goto/config": "workspace:*", "@goto/db": "workspace:*", "better-auth": "catalog:", diff --git a/apps/web/src/lib/server/visit-reports.test.ts b/apps/web/src/lib/server/visit-reports.test.ts index 2ad5d88..4f284e1 100644 --- a/apps/web/src/lib/server/visit-reports.test.ts +++ b/apps/web/src/lib/server/visit-reports.test.ts @@ -1,22 +1,18 @@ import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { + createVisitPeriod, + encodeVisitReports, + visitDays, + type OverviewSummary, + type VisitDetails +} from '@goto/analytics'; import type { GotoConfiguration } from '@goto/config'; import { openDb, runMigrations } from '@goto/db'; -import { publishVisitReports, type PreparedVisitReport } from '@goto/db/visit-reports'; +import { publishVisitReports } from '@goto/db/visit-reports'; import { expect, test, vi } from 'vitest'; -import { readVisitReport } from './visit-reports.js'; - -const report: PreparedVisitReport = { - totals: { total: 3, valid: 2, invalid: 1, base: 1 }, - daily: [], - weekly: [], - monthly: [], - browsers: [], - operatingSystems: [], - domainCounts: [], - linkCounts: [] -}; +import { readVisitSummary } from './visit-reports.js'; test('reads prepared output and hides snapshots from a different retention policy', async () => { const directory = await mkdtemp(join(tmpdir(), 'goto-page-reports-')); @@ -27,29 +23,40 @@ test('reads prepared output and hides snapshots from a different retention polic }, analytics: { retention_months: 3 } }; + const period = createVisitPeriod(3, Date.parse('2026-06-15T12:00:00.000Z')); + const totals = { validVisits: 2, invalidVisits: 1 }; + const summary: OverviewSummary = { + version: 1, + scope: { kind: 'overview' }, + period, + totals, + base: { validVisits: 1, invalidVisits: 0 }, + domains: [{ domain: 'go.example.test', totals }] + }; + const days = visitDays(period); + const details: VisitDetails = { + version: 1, + scope: summary.scope, + period, + daily: days.map((day, index) => ({ + ...day, + validVisits: index === days.length - 1 ? totals.validVisits : 0, + invalidVisits: index === days.length - 1 ? totals.invalidVisits : 0 + })), + browsers: [{ family: 'Other', ...totals }], + operatingSystems: [{ family: 'Other', ...totals }] + }; try { const db = await openDb(configuration.database.path); try { await runMigrations(db); - const period = { - timezone: 'UTC' as const, - retentionMonths: 3 as const, - since: 100, - asOf: 200, - days: [], - weeks: [], - months: [] - }; - await publishVisitReports(db, period, [ - { key: 'overview', data: JSON.stringify(report) }, - { key: 'empty', data: JSON.stringify(report) } - ]); + await publishVisitReports(db, encodeVisitReports([{ summary, details }])); } finally { await db.destroy(); } - expect((await readVisitReport(configuration))?.report.totals.total).toBe(3); + expect(await readVisitSummary(configuration, { kind: 'overview' })).toEqual(summary); configuration.analytics.retention_months = 9; - expect(await readVisitReport(configuration)).toBeNull(); + expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull(); } finally { await rm(directory, { recursive: true, force: true }); } @@ -67,7 +74,7 @@ test('enabled collection fails safely without creating missing storage', async ( }, analytics: { retention_months: 3 as const } }; - expect(await readVisitReport(configuration)).toBeNull(); + expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull(); expect(warning).toHaveBeenCalledExactlyOnceWith('Prepared visit reports unavailable'); await expect(access(missing)).rejects.toMatchObject({ code: 'ENOENT' }); } finally { @@ -83,7 +90,7 @@ test('disabled collection does not attempt to open missing storage', async () => database: { path: '/missing/not-used.sqlite', worker_path: '/missing/worker.sqlite' }, analytics: { retention_months: 0 as const } }; - expect(await readVisitReport(configuration)).toBeNull(); + expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull(); expect(warning).not.toHaveBeenCalled(); } finally { warning.mockRestore(); diff --git a/apps/web/src/lib/server/visit-reports.ts b/apps/web/src/lib/server/visit-reports.ts index 60f4fc9..7d812c9 100644 --- a/apps/web/src/lib/server/visit-reports.ts +++ b/apps/web/src/lib/server/visit-reports.ts @@ -1,23 +1,23 @@ +import type { ReportScope, VisitSummary } from '@goto/analytics'; import type { GotoConfiguration } from '@goto/config'; import { openDb } from '@goto/db'; -import { readPreparedVisitReports, type StoredVisitReports } from '@goto/db/visit-reports'; +import { readVisitSummary as readPreparedSummary } from '@goto/db/visit-reports'; import type { Immutable } from './snapshot.js'; type ReportConfiguration = Immutable>; -export async function readVisitReport( +export async function readVisitSummary( configuration: ReportConfiguration, - domain?: string, - path?: string -): Promise { + scope: ReportScope +): Promise { const months = configuration.analytics.retention_months; if (months === 0) return null; let db: Awaited> | undefined; try { db = await openDb(configuration.database.path, { readOnly: true }); - const reports = await readPreparedVisitReports(db, domain, path); - if (!reports || reports.period.retentionMonths !== months) return null; - return reports; + const summary = await readPreparedSummary(db, scope); + if (!summary || summary.period.retentionMonths !== months) return null; + return summary; } catch { warnUnavailable(); return null; diff --git a/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts index 2ae8e84..95b3a99 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts @@ -1,5 +1,5 @@ -import { readVisitReport } from '#lib/server/visit-reports.js'; -import type { StoredVisitReports } from '@goto/db/visit-reports'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; +import type { OverviewSummary } from '@goto/analytics'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = async ({ locals }) => { @@ -7,25 +7,27 @@ export const load: PageServerLoad = async ({ locals }) => { const configuration = snapshot.configuration; const retentionMonths = configuration.analytics.retention_months; const collectionEnabled = retentionMonths !== 0; - let preparedReports: StoredVisitReports | null = null; - if (collectionEnabled) preparedReports = await readVisitReport(configuration); - const report = preparedReports?.report; - + let overview: OverviewSummary | null = null; + if (collectionEnabled) { + const summary = await readVisitSummary(configuration, { kind: 'overview' }); + if (summary && 'domains' in summary) overview = summary; + } + const countsByDomain = new Map(overview?.domains.map((row) => [row.domain, row.totals])); const domains = Object.entries(snapshot.domains).map(([domain, prepared]) => ({ domain, linkCount: Object.keys(prepared.links).length, - counts: report?.domainCounts.find((row) => row.domain === domain)?.counts ?? null, + counts: countsByDomain.get(domain) ?? null, href: `/admin/domains/${encodeURIComponent(domain)}` })); + const asOf = overview ? Date.parse(overview.period.asOf) : null; return { collectionEnabled, - available: preparedReports !== null, + available: overview !== null, retentionMonths, - since: preparedReports?.period.since ?? null, - asOf: preparedReports?.period.asOf ?? null, + asOf, domainCount: domains.length, linkCount: domains.reduce((sum, row) => sum + row.linkCount, 0), - totals: report?.totals ?? null, + totals: overview?.totals ?? null, domains }; }; diff --git a/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte index 34de92e..e8ccc36 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte +++ b/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte @@ -12,7 +12,7 @@
- +
@@ -31,7 +31,7 @@ {row.domain} {formatCount(row.linkCount)} - {formatCount(row.counts?.valid ?? null)} + {formatCount(row.counts?.validVisits ?? null)} {/each} diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts index c237cda..48d27f5 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts @@ -1,7 +1,7 @@ import { error } from '@sveltejs/kit'; -import type { VisitCounts, StoredVisitReports } from '@goto/db/visit-reports'; +import type { DomainSummary, VisitTotals } from '@goto/analytics'; import { currentLinkState } from '#lib/server/link-state.js'; -import { readVisitReport } from '#lib/server/visit-reports.js'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = async ({ locals, params }) => { @@ -11,15 +11,14 @@ export const load: PageServerLoad = async ({ locals, params }) => { const prepared = snapshot.domains[domain]!; const configuration = snapshot.configuration; const collectionEnabled = configuration.analytics.retention_months !== 0; - let preparedReports: StoredVisitReports | null = null; - if (collectionEnabled) preparedReports = await readVisitReport(configuration, domain); - const report = preparedReports?.report; - - const countsByPath = new Map(); - for (const row of report?.linkCounts ?? []) countsByPath.set(row.path, row.counts); - const baseCounts = countsByPath.get('/') ?? null; - let invalidClicks: number | null = null; - if (report && baseCounts) invalidClicks = report.totals.invalid - baseCounts.invalid; + let domainSummary: DomainSummary | null = null; + if (collectionEnabled) { + const scope = { kind: 'domain' as const, domain }; + const summary = await readVisitSummary(configuration, scope); + if (summary && 'links' in summary) domainSummary = summary; + } + const countsByPath = new Map(); + for (const row of domainSummary?.links ?? []) countsByPath.set(row.path, row.totals); const now = Date.now(); const links = Object.entries(prepared.links).map(([path, link]) => ({ path, @@ -35,21 +34,25 @@ export const load: PageServerLoad = async ({ locals, params }) => { baseUrl = root.url; baseState = 'Active'; } + let invalidClicks: number | null = null; + if (domainSummary) { + invalidClicks = domainSummary.totals.invalidVisits - domainSummary.base.invalidVisits; + } + const asOf = domainSummary ? Date.parse(domainSummary.period.asOf) : null; return { domain, retentionMonths: configuration.analytics.retention_months, collectionEnabled, - available: preparedReports !== null, - since: preparedReports?.period.since ?? null, - asOf: preparedReports?.period.asOf ?? null, + available: domainSummary !== null, + asOf, linkCount: links.length, - totals: report?.totals ?? null, + totals: domainSummary?.totals ?? null, invalidClicks, base: { url: baseUrl, state: baseState, - counts: baseCounts, + counts: domainSummary?.base ?? null, href: linkHref(domain, '/') }, links diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte index b622bec..a9043f4 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte @@ -11,7 +11,7 @@
- +
@@ -35,7 +35,7 @@ >{:else}404 Not found{/if} {data.base.state} - {formatCount(data.base.counts?.valid ?? null)} + {formatCount(data.base.counts?.validVisits ?? null)} @@ -60,7 +60,7 @@ >{link.path} {link.url} - {link.state}{formatCount(link.counts?.valid ?? null)} + {link.state}{formatCount(link.counts?.validVisits ?? null)} {/each} diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts index c5baf33..a52c8c3 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts @@ -1,7 +1,7 @@ import { error } from '@sveltejs/kit'; -import type { StoredVisitReports } from '@goto/db/visit-reports'; +import type { VisitSummary } from '@goto/analytics'; import { currentLinkState } from '#lib/server/link-state.js'; -import { readVisitReport } from '#lib/server/visit-reports.js'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; import type { PageServerLoad } from './$types'; export const load: PageServerLoad = async ({ locals, params }) => { @@ -15,9 +15,13 @@ export const load: PageServerLoad = async ({ locals, params }) => { const link = prepared.links[path]; const configuration = snapshot.configuration; const collectionEnabled = configuration.analytics.retention_months !== 0; - let reports: StoredVisitReports | null = null; - if (collectionEnabled) reports = await readVisitReport(configuration, domain, path); + let linkSummary: VisitSummary | null = null; + if (collectionEnabled) { + const scope = { kind: 'link' as const, domain, path }; + linkSummary = await readVisitSummary(configuration, scope); + } const state = link ? currentLinkState(link, Date.now()) : 'Fallback'; + const asOf = linkSummary ? Date.parse(linkSummary.period.asOf) : null; return { domain, @@ -28,9 +32,8 @@ export const load: PageServerLoad = async ({ locals, params }) => { state, retentionMonths: configuration.analytics.retention_months, collectionEnabled, - available: reports !== null, - since: reports?.period.since ?? null, - asOf: reports?.period.asOf ?? null, - totals: reports?.selection.totals ?? null + available: linkSummary !== null, + asOf, + totals: linkSummary?.totals ?? null }; }; diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte index e1b52dd..a5050fd 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte @@ -16,7 +16,7 @@ >
- +
diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts index 10d1679..9784656 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts @@ -1,11 +1,11 @@ import { validateConfig } from '@goto/config'; -import type { PreparedVisitReport } from '@goto/db/visit-reports'; +import type { LinkSummary } from '@goto/analytics'; import { beforeEach, expect, test, vi } from 'vitest'; import { prepareSnapshot } from '#lib/server/snapshot.js'; -import { readVisitReport } from '#lib/server/visit-reports.js'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; import { load } from './+page.server.js'; -vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() })); beforeEach(() => vi.resetAllMocks()); const domain = 'go.example.test'; @@ -31,47 +31,41 @@ function request(path = '/docs', host = domain, retention_months = 3) { params: { domain: host, path: path.slice(1) } } as Parameters[0]; } -const report: PreparedVisitReport = { - totals: { total: 6, valid: 3, invalid: 3, base: 1 }, - daily: [], - weekly: [], - monthly: [], - browsers: [], - operatingSystems: [], - domainCounts: [], - linkCounts: [] -}; -const period = { - timezone: 'UTC' as const, - retentionMonths: 3 as const, - since: 100, - asOf: 200, - days: [], - weeks: [], - months: [] +const summary: LinkSummary = { + version: 1, + scope: { kind: 'link', domain, path: '/docs' }, + period: { + from: '2026-03-15T12:00:00.000Z', + asOf: '2026-06-15T12:00:00.000Z', + timezone: 'UTC', + retentionMonths: 3 + }, + totals: { validVisits: 2, invalidVisits: 1 } }; -test('loads the requested path report and current destination with a domain breadcrumb', async () => { - const totals = { total: 3, valid: 2, invalid: 1, base: 0 }; - vi.mocked(readVisitReport).mockResolvedValue({ - period, - report, - selection: { ...report, totals } - }); +test('loads the requested path summary and current destination with a domain breadcrumb', async () => { + vi.mocked(readVisitSummary).mockResolvedValue(summary); const data = await load(request()); if (!data) throw new Error('Missing link page'); - expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/docs'); + expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), { + kind: 'link', + domain, + path: '/docs' + }); expect(data.path).toBe('/docs'); - expect(data.totals).toEqual(totals); + expect(data.totals).toEqual(summary.totals); expect(data.url).toBe('https://example.test/docs'); expect(data.state).toBe('Active'); expect(data.domainHref).toBe('/admin/domains/go.example.test'); - expect(data.asOf).toBe(200); + expect(data.asOf).toBe(Date.parse(summary.period.asOf)); expect(data.retentionMonths).toBe(3); }); test('keeps decoded nested and literal-percent paths distinct and supports the unconfigured root', async () => { - vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report }); + vi.mocked(readVisitSummary) + .mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/a/b' } }) + .mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/a%2Fb' } }) + .mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/' } }); const nested = await load(request('/a/b')); const percent = await load(request('/a%2Fb')); const root = await load(request('/')); @@ -79,7 +73,21 @@ test('keeps decoded nested and literal-percent paths distinct and supports the u expect(nested.path).toBe('/a/b'); expect(percent.path).toBe('/a%2Fb'); expect(percent.state).toBe('Disabled'); - expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/a%2Fb'); + expect(readVisitSummary).toHaveBeenNthCalledWith(1, expect.any(Object), { + kind: 'link', + domain, + path: '/a/b' + }); + expect(readVisitSummary).toHaveBeenNthCalledWith(2, expect.any(Object), { + kind: 'link', + domain, + path: '/a%2Fb' + }); + expect(readVisitSummary).toHaveBeenNthCalledWith(3, expect.any(Object), { + kind: 'link', + domain, + path: '/' + }); expect(root.path).toBe('/'); expect(root.state).toBe('Fallback'); expect(root.url).toBeNull(); @@ -89,8 +97,8 @@ test('keeps decoded nested and literal-percent paths distinct and supports the u test('rejects unknown scopes before reading and keeps unavailable or disabled counts empty', async () => { await expect(load(request('/docs', 'unknown.test'))).rejects.toMatchObject({ status: 404 }); await expect(load(request('/not-configured'))).rejects.toMatchObject({ status: 404 }); - expect(readVisitReport).not.toHaveBeenCalled(); - vi.mocked(readVisitReport).mockResolvedValue(null); + expect(readVisitSummary).not.toHaveBeenCalled(); + vi.mocked(readVisitSummary).mockResolvedValue(null); const missing = await load(request()); if (!missing) throw new Error('Missing unavailable page'); expect(missing.available).toBe(false); @@ -101,5 +109,5 @@ test('rejects unknown scopes before reading and keeps unavailable or disabled co expect(disabled.collectionEnabled).toBe(false); expect(disabled.retentionMonths).toBe(0); expect(disabled.totals).toBeNull(); - expect(readVisitReport).not.toHaveBeenCalled(); + expect(readVisitSummary).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts index fc3ea2d..8fd1391 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts @@ -1,11 +1,11 @@ import { validateConfig } from '@goto/config'; -import type { PreparedVisitReport, StoredVisitReports } from '@goto/db/visit-reports'; +import type { DomainSummary } from '@goto/analytics'; import { beforeEach, expect, test, vi } from 'vitest'; import { prepareSnapshot } from '#lib/server/snapshot.js'; -import { readVisitReport } from '#lib/server/visit-reports.js'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; import { load } from './+page.server.js'; -vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() })); beforeEach(() => vi.resetAllMocks()); const domain = 'go.example.test'; @@ -28,59 +28,53 @@ function request(host = domain, rootUrl?: string) { params: { domain: host } } as Parameters[0]; } -const docs = { total: 3, valid: 2, invalid: 1, base: 0 }; -const root = { total: 1, valid: 0, invalid: 1, base: 1 }; -const empty = { total: 0, valid: 0, invalid: 0, base: 0 }; -const report: PreparedVisitReport = { - totals: { total: 6, valid: 3, invalid: 3, base: 1 }, - daily: [], - weekly: [], - monthly: [], - browsers: [], - operatingSystems: [], - domainCounts: [], - linkCounts: [ - { path: '/docs', counts: docs }, - { path: '/', counts: root }, - { path: '/a%2Fb', counts: empty } +const docs = { validVisits: 2, invalidVisits: 1 }; +const root = { validVisits: 0, invalidVisits: 1 }; +const empty = { validVisits: 0, invalidVisits: 0 }; +const summary: DomainSummary = { + version: 1, + scope: { kind: 'domain', domain }, + period: { + from: '2026-03-15T12:00:00.000Z', + asOf: '2026-06-15T12:00:00.000Z', + timezone: 'UTC', + retentionMonths: 3 + }, + totals: { validVisits: 3, invalidVisits: 3 }, + base: root, + links: [ + { path: '/docs', totals: docs }, + { path: '/', totals: root }, + { path: '/a%2Fb', totals: empty } ] }; -const period = { - timezone: 'UTC' as const, - retentionMonths: 3 as const, - since: 100, - asOf: 200, - days: [], - weeks: [], - months: [] -}; test('reads only domain-wide traffic and links to dedicated path routes', async () => { - const reports: StoredVisitReports = { period, report, selection: report }; - vi.mocked(readVisitReport).mockResolvedValue(reports); + vi.mocked(readVisitSummary).mockResolvedValue(summary); const data = await load(request()); if (!data) throw new Error('Missing page data'); - expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain); - expect(data.totals).toEqual(report.totals); + expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), { + kind: 'domain', + domain + }); + expect(data.totals).toEqual(summary.totals); expect(data.invalidClicks).toBe(2); expect(data.linkCount).toBe(3); expect(data.retentionMonths).toBe(3); expect(data.links).toMatchObject([ - { href: '/admin/domains/go.example.test/links/docs' }, + { href: '/admin/domains/go.example.test/links/docs', counts: docs }, { href: '/admin/domains/go.example.test/links/a/b' }, - { href: '/admin/domains/go.example.test/links/a%252Fb' } + { href: '/admin/domains/go.example.test/links/a%252Fb', counts: empty, state: 'Disabled' } ]); - expect(data.links[2]?.counts).toEqual(empty); - expect(data.links[2]?.state).toBe('Disabled'); expect(data.base.href).toBe('/admin/domains/go.example.test/links'); expect(data.base.url).toBeNull(); expect(data.base.state).toBe('Fallback'); expect(data.base.counts).toEqual(root); - expect(data.asOf).toBe(200); + expect(data.asOf).toBe(Date.parse(summary.period.asOf)); }); test('shows an active base destination without reclassifying historical outcomes', async () => { - vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report }); + vi.mocked(readVisitSummary).mockResolvedValue(summary); const data = await load(request(domain, 'https://example.test/start')); if (!data) throw new Error('Missing base page data'); expect(data.base.url).toBe('https://example.test/start'); @@ -89,29 +83,24 @@ test('shows an active base destination without reclassifying historical outcomes }); test('excludes invalid base traffic without subtracting valid base redirects', async () => { - const mixedRoot = { total: 5, valid: 3, invalid: 2, base: 5 }; - const mixedReport: PreparedVisitReport = { - ...report, - totals: { total: 10, valid: 6, invalid: 4, base: 5 }, - linkCounts: report.linkCounts.map((row) => - row.path === '/' ? { ...row, counts: mixedRoot } : row - ) + const mixedRoot = { validVisits: 3, invalidVisits: 2 }; + const mixedSummary: DomainSummary = { + ...summary, + totals: { validVisits: 6, invalidVisits: 4 }, + base: mixedRoot, + links: summary.links.map((row) => (row.path === '/' ? { ...row, totals: mixedRoot } : row)) }; - vi.mocked(readVisitReport).mockResolvedValue({ - period, - report: mixedReport, - selection: mixedReport - }); + vi.mocked(readVisitSummary).mockResolvedValue(mixedSummary); const data = await load(request()); if (!data) throw new Error('Missing domain page data'); expect(data.invalidClicks).toBe(2); - expect(data.totals?.valid).toBe(6); + expect(data.totals?.validVisits).toBe(6); }); test('rejects unknown domains before reading and preserves unavailable rather than zero', async () => { await expect(load(request('unknown.test'))).rejects.toMatchObject({ status: 404 }); - expect(readVisitReport).not.toHaveBeenCalled(); - vi.mocked(readVisitReport).mockResolvedValue(null); + expect(readVisitSummary).not.toHaveBeenCalled(); + vi.mocked(readVisitSummary).mockResolvedValue(null); const data = await load(request()); if (!data) throw new Error('Missing unavailable page'); expect(data.available).toBe(false); diff --git a/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts index 3991596..72c2253 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts @@ -1,11 +1,11 @@ import { validateConfig } from '@goto/config'; -import type { StoredVisitReports } from '@goto/db/visit-reports'; +import type { OverviewSummary } from '@goto/analytics'; import { beforeEach, expect, test, vi } from 'vitest'; import { prepareSnapshot } from '#lib/server/snapshot.js'; -import { readVisitReport } from '#lib/server/visit-reports.js'; +import { readVisitSummary } from '#lib/server/visit-reports.js'; import { load } from './+page.server.js'; -vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() })); beforeEach(() => vi.resetAllMocks()); function request(retention_months = 3) { @@ -27,39 +27,35 @@ function request(retention_months = 3) { return { locals: { snapshot: prepareSnapshot(prepared) } } as Parameters[0]; } -test('shows current configuration and the worker-prepared snapshot without re-aggregating', async () => { - const counts = { total: 4, valid: 2, invalid: 2, base: 1 }; - const report = { - totals: { ...counts, total: 9, valid: 7 }, - daily: [], - weekly: [], - monthly: [], - browsers: [], - operatingSystems: [], - domainCounts: [{ domain: 'go.example.test', counts }], - linkCounts: [] - }; - const prepared: StoredVisitReports = { +test('shows current configuration and the worker-prepared summary without re-aggregating', async () => { + const counts = { validVisits: 2, invalidVisits: 2 }; + const summary: OverviewSummary = { + version: 1, + scope: { kind: 'overview' }, period: { + from: '2026-03-15T12:00:00.000Z', + asOf: '2026-06-15T12:00:00.000Z', timezone: 'UTC', - retentionMonths: 3, - since: 100, - asOf: 200, - days: [], - weeks: [], - months: [] + retentionMonths: 3 }, - report, - selection: report + totals: { validVisits: 7, invalidVisits: 2 }, + base: { validVisits: 0, invalidVisits: 1 }, + domains: [{ domain: 'go.example.test', totals: counts }] }; - vi.mocked(readVisitReport).mockResolvedValue(prepared); + vi.mocked(readVisitSummary).mockResolvedValue(summary); const data = await load(request()); if (!data) throw new Error('Missing page data'); + expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), { + kind: 'overview' + }); expect(data.domainCount).toBe(1); expect(data.linkCount).toBe(2); - expect(data.totals?.total).toBe(9); - expect(data.asOf).toBe(200); - expect(data.domains[0]?.href).toBe('/admin/domains/go.example.test'); + expect(data.totals).toEqual(summary.totals); + expect(data.asOf).toBe(Date.parse(summary.period.asOf)); + expect(data.domains[0]).toMatchObject({ + href: '/admin/domains/go.example.test', + counts + }); }); test('disabled collection does not read a report; missing preparation is not zero', async () => { @@ -67,8 +63,8 @@ test('disabled collection does not read a report; missing preparation is not zer if (!disabled) throw new Error('Missing disabled page'); expect(disabled.collectionEnabled).toBe(false); expect(disabled.totals).toBeNull(); - expect(readVisitReport).not.toHaveBeenCalled(); - vi.mocked(readVisitReport).mockResolvedValue(null); + expect(readVisitSummary).not.toHaveBeenCalled(); + vi.mocked(readVisitSummary).mockResolvedValue(null); const unavailable = await load(request()); if (!unavailable) throw new Error('Missing unavailable page'); expect(unavailable.available).toBe(false); diff --git a/apps/worker/package.json b/apps/worker/package.json index 2c06bd2..9018fc9 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -18,6 +18,7 @@ "test:smoke": "node e2e/startup.js" }, "dependencies": { + "@goto/analytics": "workspace:*", "@goto/config": "workspace:*", "@goto/db": "workspace:*", "openworkflow": "catalog:", diff --git a/apps/worker/src/services/build-visit-reports.test.ts b/apps/worker/src/services/build-visit-reports.test.ts index 332a817..3dcd181 100644 --- a/apps/worker/src/services/build-visit-reports.test.ts +++ b/apps/worker/src/services/build-visit-reports.test.ts @@ -1,47 +1,233 @@ -import type { PreparedVisitReport } from '@goto/db/visit-reports'; +import { createVisitPeriod, visitDays } from '@goto/analytics'; import { expect, test } from 'vitest'; import { buildVisitReports } from './build-visit-reports.js'; -import { reportPeriod } from './report-buckets.js'; -test('places counts on the correct UTC days, Monday weeks and calendar months', () => { - const period = reportPeriod(3, Date.parse('2026-07-01T12:00:00Z')); - const reports = buildVisitReports( +test('builds gap-free UTC daily details with zeros and partial calendar edges', () => { + const period = createVisitPeriod(3, Date.parse('2026-05-31T12:34:56.789Z')); + const [report] = buildVisitReports( { daily: [ - { scope: 'overview', day: '2026-05-31', total: 3, valid: 2, invalid: 1, base: 1 }, - { scope: 'overview', day: '2026-06-01', total: 5, valid: 1, invalid: 4, base: 2 } + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-02-28', + validVisits: 2, + invalidVisits: 1, + baseValidVisits: 1, + baseInvalidVisits: 0 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-05-31', + validVisits: 1, + invalidVisits: 4, + baseValidVisits: 0, + baseInvalidVisits: 2 + } ], - browsers: [], - operatingSystems: [] + browsers: [ + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Firefox', + validVisits: 3, + invalidVisits: 5 + } + ], + operatingSystems: [ + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Linux', + validVisits: 3, + invalidVisits: 5 + } + ] }, [], period ); - const report = JSON.parse( - reports.find((entry) => entry.key === 'overview')!.data - ) as PreparedVisitReport; - const firstDay = period.days.findIndex( - (bucket) => bucket.start === Date.parse('2026-05-30T00:00:00Z') - ); - const firstWeek = period.weeks.findIndex( - (bucket) => bucket.start === Date.parse('2026-05-25T00:00:00Z') - ); - expect(report.daily.slice(firstDay, firstDay + 4)).toEqual([ - { total: 0, valid: 0, invalid: 0, base: 0 }, - { total: 3, valid: 2, invalid: 1, base: 1 }, - { total: 5, valid: 1, invalid: 4, base: 2 }, - { total: 0, valid: 0, invalid: 0, base: 0 } + expect(report?.summary).toEqual({ + version: 1, + scope: { kind: 'overview' }, + period, + totals: { validVisits: 3, invalidVisits: 5 }, + base: { validVisits: 1, invalidVisits: 2 }, + domains: [] + }); + expect(report?.details.daily).toHaveLength(visitDays(period).length); + expect(report?.details.daily[0]).toEqual({ + start: '2026-02-28T00:00:00.000Z', + end: '2026-03-01T00:00:00.000Z', + partial: true, + validVisits: 2, + invalidVisits: 1 + }); + expect(report?.details.daily[1]).toEqual({ + start: '2026-03-01T00:00:00.000Z', + end: '2026-03-02T00:00:00.000Z', + partial: false, + validVisits: 0, + invalidVisits: 0 + }); + expect(report?.details.daily.at(-1)).toEqual({ + start: '2026-05-31T00:00:00.000Z', + end: '2026-06-01T00:00:00.000Z', + partial: true, + validVisits: 1, + invalidVisits: 4 + }); +}); + +test('indexes explicit domain and link scopes, including empty links and root, without mixing families', () => { + const period = createVisitPeriod(3, Date.parse('2026-07-01T12:00:00Z')); + const overview = { scopeKind: 'overview' as const, domain: null, path: null }; + const domain = { scopeKind: 'domain' as const, domain: 'go.example.test', path: null }; + const link = { scopeKind: 'link' as const, domain: 'go.example.test', path: '/docs' }; + const root = { ...link, path: '/' }; + const reports = buildVisitReports( + { + daily: [ + { + ...overview, + day: '2026-06-01', + validVisits: 8, + invalidVisits: 3, + baseValidVisits: 1, + baseInvalidVisits: 1 + }, + { + ...domain, + day: '2026-06-01', + validVisits: 4, + invalidVisits: 2, + baseValidVisits: 1, + baseInvalidVisits: 1 + }, + { + ...link, + day: '2026-06-01', + validVisits: 2, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + ...root, + day: '2026-06-01', + validVisits: 1, + invalidVisits: 1, + baseValidVisits: 1, + baseInvalidVisits: 1 + } + ], + browsers: [ + { ...overview, family: 'Other', validVisits: 8, invalidVisits: 3 }, + { ...domain, family: 'Firefox', validVisits: 4, invalidVisits: 2 }, + { ...link, family: 'Chrome', validVisits: 2, invalidVisits: 0 }, + { ...root, family: 'Firefox', validVisits: 1, invalidVisits: 1 } + ], + operatingSystems: [ + { ...overview, family: 'Other', validVisits: 8, invalidVisits: 3 }, + { ...domain, family: 'Linux', validVisits: 4, invalidVisits: 2 }, + { ...link, family: 'Linux', validVisits: 2, invalidVisits: 0 }, + { ...root, family: 'Linux', validVisits: 1, invalidVisits: 1 } + ] + }, + [ + { domain: 'go.example.test', paths: ['/docs', '/empty', '/'] }, + { domain: 'empty.example.test', paths: ['/docs'] } + ], + period + ); + const docs = reports.find( + ({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/docs' + ); + const base = reports.find( + ({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/' + ); + const empty = reports.find( + ({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/empty' + ); + const domainReport = reports.find(({ summary }) => summary.scope.kind === 'domain'); + const overviewReport = reports.find(({ summary }) => summary.scope.kind === 'overview'); + + expect(reports).toHaveLength(8); + expect(domainReport?.summary).toEqual({ + version: 1, + scope: { kind: 'domain', domain: 'go.example.test' }, + period, + totals: { validVisits: 4, invalidVisits: 2 }, + base: { validVisits: 1, invalidVisits: 1 }, + links: [ + { path: '/docs', totals: { validVisits: 2, invalidVisits: 0 } }, + { path: '/empty', totals: { validVisits: 0, invalidVisits: 0 } }, + { path: '/', totals: { validVisits: 1, invalidVisits: 1 } } + ] + }); + expect(overviewReport?.summary).toEqual({ + version: 1, + scope: { kind: 'overview' }, + period, + totals: { validVisits: 8, invalidVisits: 3 }, + base: { validVisits: 1, invalidVisits: 1 }, + domains: [ + { domain: 'go.example.test', totals: { validVisits: 4, invalidVisits: 2 } }, + { domain: 'empty.example.test', totals: { validVisits: 0, invalidVisits: 0 } } + ] + }); + expect(docs?.summary).toEqual({ + version: 1, + scope: { kind: 'link', domain: 'go.example.test', path: '/docs' }, + period, + totals: { validVisits: 2, invalidVisits: 0 } + }); + expect(docs?.details.browsers).toEqual([{ family: 'Chrome', validVisits: 2, invalidVisits: 0 }]); + expect(domainReport?.details.browsers).toEqual([ + { family: 'Firefox', validVisits: 4, invalidVisits: 2 } ]); - expect(report.weekly.slice(firstWeek, firstWeek + 2)).toEqual([ - { total: 3, valid: 2, invalid: 1, base: 1 }, - { total: 5, valid: 1, invalid: 4, base: 2 } + expect(base?.details.operatingSystems).toEqual([ + { family: 'Linux', validVisits: 1, invalidVisits: 1 } ]); - expect(report.monthly).toEqual([ - { total: 0, valid: 0, invalid: 0, base: 0 }, - { total: 3, valid: 2, invalid: 1, base: 1 }, - { total: 5, valid: 1, invalid: 4, base: 2 }, - { total: 0, valid: 0, invalid: 0, base: 0 } - ]); - expect(report.totals).toEqual({ total: 8, valid: 3, invalid: 5, base: 3 }); + expect(empty?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 }); + expect(empty?.details.daily).toEqual( + visitDays(period).map((day) => ({ ...day, validVisits: 0, invalidVisits: 0 })) + ); + expect(empty?.details.browsers).toEqual([]); + const otherDomainDocs = reports.find( + ({ summary }) => + summary.scope.kind === 'link' && + summary.scope.domain === 'empty.example.test' && + summary.scope.path === '/docs' + ); + expect(otherDomainDocs?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 }); + const otherDomain = reports.find( + ({ summary }) => + summary.scope.kind === 'domain' && summary.scope.domain === 'empty.example.test' + ); + expect(otherDomain?.summary).toEqual({ + version: 1, + scope: { kind: 'domain', domain: 'empty.example.test' }, + period, + totals: { validVisits: 0, invalidVisits: 0 }, + base: { validVisits: 0, invalidVisits: 0 }, + links: [{ path: '/docs', totals: { validVisits: 0, invalidVisits: 0 } }] + }); + const unconfiguredRoot = reports.find( + ({ summary }) => + summary.scope.kind === 'link' && + summary.scope.domain === 'empty.example.test' && + summary.scope.path === '/' + ); + expect(unconfiguredRoot?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 }); + for (const report of reports) { + expect(report.details.scope).toEqual(report.summary.scope); + expect(report.details.period).toEqual(period); + } }); diff --git a/apps/worker/src/services/build-visit-reports.ts b/apps/worker/src/services/build-visit-reports.ts index dcfe735..c4a3f23 100644 --- a/apps/worker/src/services/build-visit-reports.ts +++ b/apps/worker/src/services/build-visit-reports.ts @@ -1,14 +1,20 @@ import { - visitReportKey, - zeroVisitCounts, - type AggregatedVisitReports, - type DailyVisitCounts, - type ScopedFamilyCounts, + visitDays, + zeroVisitTotals, + type DomainSummary, + type LinkSummary, + type OverviewSummary, type PreparedVisitReport, - type ReportInventory, - type SerializedVisitReport, - type VisitCounts, - type VisitReportPeriod + type ReportPeriod, + type ReportScope, + type VisitDetails, + type VisitTotals +} from '@goto/analytics'; +import type { + AggregatedVisitReports, + DailyVisitCounts, + ReportInventory, + ScopedFamilyCounts } from '@goto/db/visit-reports'; interface ScopeCounts { @@ -17,117 +23,164 @@ interface ScopeCounts { operatingSystems: ScopedFamilyCounts[]; } +interface DomainCounts { + counts: ScopeCounts; + links: Map; +} + +interface ReportCounts { + overview: ScopeCounts; + domains: Map; +} + export function buildVisitReports( aggregates: AggregatedVisitReports, inventory: ReportInventory, - period: VisitReportPeriod -): SerializedVisitReport[] { + period: ReportPeriod +): PreparedVisitReport[] { const indexed = indexCounts(aggregates); - const empty = emptyReport(period); - const reports: SerializedVisitReport[] = [{ key: 'empty', data: JSON.stringify(empty) }]; - const linkTotals = new Map(); - const domainTotals = new Map(); + const days = visitDays(period); + const reports: PreparedVisitReport[] = []; + const domains: OverviewSummary['domains'] = []; for (const entry of inventory) { - const paths = [...new Set([...entry.paths, '/'])]; - for (const path of paths) { - const key = visitReportKey(entry.domain, path); - const source = indexed.get(key); - if (!source) { - linkTotals.set(key, zeroVisitCounts()); - continue; - } - const report = scopeReport(source, period); - linkTotals.set(key, report.totals); - reports.push({ key, data: JSON.stringify(report) }); + const domain = indexed.domains.get(entry.domain); + const links: DomainSummary['links'] = []; + const configuredPaths = new Set(entry.paths); + const allPaths = new Set([...configuredPaths, '/']); + for (const path of allPaths) { + const counts = domain?.links.get(path) ?? emptyCounts(); + const scope = { kind: 'link' as const, domain: entry.domain, path }; + const summary = buildLinkSummary(scope, counts, period); + if (configuredPaths.has(path)) links.push({ path, totals: summary.totals }); + reports.push({ summary, details: buildDetails(scope, counts, period, days) }); } - const key = visitReportKey(entry.domain); - const source = indexed.get(key); - const report = source ? scopeReport(source, period) : { ...empty }; - report.linkCounts = paths.map((path) => { - const linkKey = visitReportKey(entry.domain, path); - return { path, counts: linkTotals.get(linkKey) ?? zeroVisitCounts() }; - }); - domainTotals.set(entry.domain, report.totals); - reports.push({ key, data: JSON.stringify(report) }); + const scope = { kind: 'domain' as const, domain: entry.domain }; + const counts = domain?.counts ?? emptyCounts(); + const summary = buildDomainSummary(scope, counts, links, period); + domains.push({ domain: entry.domain, totals: summary.totals }); + reports.push({ summary, details: buildDetails(scope, counts, period, days) }); } - const overviewSource = indexed.get('overview'); - const overview = overviewSource ? scopeReport(overviewSource, period) : { ...empty }; - overview.domainCounts = inventory.map((entry) => ({ - domain: entry.domain, - counts: domainTotals.get(entry.domain) ?? zeroVisitCounts() - })); - reports.push({ key: 'overview', data: JSON.stringify(overview) }); + + const summary = buildOverviewSummary(indexed.overview, domains, period); + reports.push({ + summary, + details: buildDetails(summary.scope, indexed.overview, period, days) + }); return reports; } -function indexCounts(aggregates: AggregatedVisitReports): Map { - const indexed = new Map(); - function scope(key: string): ScopeCounts { - let counts = indexed.get(key); - if (!counts) { - counts = { daily: [], browsers: [], operatingSystems: [] }; - indexed.set(key, counts); - } - return counts; - } - for (const row of aggregates.daily) scope(row.scope).daily.push(row); - for (const row of aggregates.browsers) scope(row.scope).browsers.push(row); - for (const row of aggregates.operatingSystems) scope(row.scope).operatingSystems.push(row); +function indexCounts(aggregates: AggregatedVisitReports): ReportCounts { + const indexed: ReportCounts = { overview: emptyCounts(), domains: new Map() }; + for (const row of aggregates.daily) countsForScope(indexed, row).daily.push(row); + for (const row of aggregates.browsers) countsForScope(indexed, row).browsers.push(row); + for (const row of aggregates.operatingSystems) + countsForScope(indexed, row).operatingSystems.push(row); return indexed; } -function emptyReport(period: VisitReportPeriod): PreparedVisitReport { +function countsForScope( + indexed: ReportCounts, + scope: DailyVisitCounts | ScopedFamilyCounts +): ScopeCounts { + if (scope.scopeKind === 'overview') return indexed.overview; + let domain = indexed.domains.get(scope.domain); + if (!domain) { + domain = { counts: emptyCounts(), links: new Map() }; + indexed.domains.set(scope.domain, domain); + } + if (scope.scopeKind === 'domain') return domain.counts; + let link = domain.links.get(scope.path); + if (!link) { + link = emptyCounts(); + domain.links.set(scope.path, link); + } + return link; +} + +function emptyCounts(): ScopeCounts { + return { daily: [], browsers: [], operatingSystems: [] }; +} + +function buildOverviewSummary( + counts: ScopeCounts, + domains: OverviewSummary['domains'], + period: ReportPeriod +): OverviewSummary { return { - totals: zeroVisitCounts(), - daily: period.days.map(() => zeroVisitCounts()), - weekly: period.weeks.map(() => zeroVisitCounts()), - monthly: period.months.map(() => zeroVisitCounts()), - browsers: [], - operatingSystems: [], - domainCounts: [], - linkCounts: [] + version: 1, + scope: { kind: 'overview' }, + period, + ...summarizeDaily(counts.daily), + domains }; } -function scopeReport(source: ScopeCounts, period: VisitReportPeriod): PreparedVisitReport { - const report = emptyReport(period); - const dayIndexes = new Map(); - period.days.forEach((bucket, index) => dayIndexes.set(bucket.start, index)); - for (const row of source.daily) { - const start = Date.parse(`${row.day}T00:00:00Z`); - const index = dayIndexes.get(start); - if (index === undefined) continue; - addCounts(report.daily[index]!, row); - addCounts(report.totals, row); +function buildDomainSummary( + scope: DomainSummary['scope'], + counts: ScopeCounts, + links: DomainSummary['links'], + period: ReportPeriod +): DomainSummary { + return { version: 1, scope, period, ...summarizeDaily(counts.daily), links }; +} + +function buildLinkSummary( + scope: LinkSummary['scope'], + counts: ScopeCounts, + period: ReportPeriod +): LinkSummary { + const { totals } = summarizeDaily(counts.daily); + return { version: 1, scope, period, totals }; +} + +function summarizeDaily(rows: DailyVisitCounts[]): { totals: VisitTotals; base: VisitTotals } { + const totals = zeroVisitTotals(); + const base = zeroVisitTotals(); + for (const row of rows) { + totals.validVisits += row.validVisits; + totals.invalidVisits += row.invalidVisits; + base.validVisits += row.baseValidVisits; + base.invalidVisits += row.baseInvalidVisits; } - for (let index = 0; index < period.days.length; index++) { - const day = period.days[index]!; - const counts = report.daily[index]!; - const week = period.weeks.findIndex( - (bucket) => day.start >= bucket.start && day.start < bucket.end - ); - const month = period.months.findIndex( - (bucket) => day.start >= bucket.start && day.start < bucket.end - ); - if (week >= 0) addCounts(report.weekly[week]!, counts); - if (month >= 0) addCounts(report.monthly[month]!, counts); - } - report.browsers = source.browsers.map((row) => ({ name: row.name, counts: copyCounts(row) })); - report.operatingSystems = source.operatingSystems.map((row) => ({ - name: row.name, - counts: copyCounts(row) + return { totals, base }; +} + +function buildDetails( + scope: ReportScope, + counts: ScopeCounts, + period: ReportPeriod, + days: ReturnType +): VisitDetails { + return { + version: 1, + scope, + period, + daily: buildDailyDetails(counts.daily, days), + browsers: buildFamilyDetails(counts.browsers), + operatingSystems: buildFamilyDetails(counts.operatingSystems) + }; +} + +function buildDailyDetails( + rows: DailyVisitCounts[], + days: ReturnType +): VisitDetails['daily'] { + const countsByDay = new Map(rows.map((row) => [row.day, row])); + return days.map((day) => { + const counts = countsByDay.get(day.start.slice(0, 10)); + return { + ...day, + validVisits: counts?.validVisits ?? 0, + invalidVisits: counts?.invalidVisits ?? 0 + }; + }); +} + +function buildFamilyDetails(rows: ScopedFamilyCounts[]): VisitDetails['browsers'] { + return rows.map(({ family, validVisits, invalidVisits }) => ({ + family, + validVisits, + invalidVisits })); - return report; -} - -function copyCounts(counts: VisitCounts): VisitCounts { - return { total: counts.total, valid: counts.valid, invalid: counts.invalid, base: counts.base }; -} - -function addCounts(target: VisitCounts, counts: VisitCounts): void { - target.total += counts.total; - target.valid += counts.valid; - target.invalid += counts.invalid; - target.base += counts.base; } diff --git a/apps/worker/src/services/prepare-visit-reports.test.ts b/apps/worker/src/services/prepare-visit-reports.test.ts index 748da98..1470076 100644 --- a/apps/worker/src/services/prepare-visit-reports.test.ts +++ b/apps/worker/src/services/prepare-visit-reports.test.ts @@ -1,6 +1,7 @@ +import { encodeVisitReports, type VisitTotals } from '@goto/analytics'; import { validateConfig } from '@goto/config'; import { openDb, runMigrations, insertVisit, pruneVisits, type ClassifiedVisit } from '@goto/db'; -import { readPreparedVisitReports } from '@goto/db/visit-reports'; +import { readVisitSummary, readVisitDetails } from '@goto/db/visit-reports'; import { afterEach, expect, test, vi } from 'vitest'; import { prepareVisitReports } from './prepare-visit-reports.js'; import type { WorkerContext } from './context.js'; @@ -10,7 +11,15 @@ vi.mock('@goto/db', async (importOriginal) => { return { ...actual, pruneVisits: vi.fn(actual.pruneVisits) }; }); -afterEach(() => vi.mocked(pruneVisits).mockReset()); +vi.mock('@goto/analytics', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, encodeVisitReports: vi.fn(actual.encodeVisitReports) }; +}); + +afterEach(() => { + vi.mocked(pruneVisits).mockReset(); + vi.mocked(encodeVisitReports).mockReset(); +}); async function context(): Promise { const prepared = validateConfig( @@ -35,12 +44,21 @@ const visit: ClassifiedVisit = { browser: 'Firefox', os: 'Linux' }; +const overview = { kind: 'overview' as const }; +const domain = { kind: 'domain' as const, domain: 'go.example.test' }; +const docs = { kind: 'link' as const, domain: 'go.example.test', path: '/docs' }; -function sum(values: { total: number }[]): number { - return values.reduce((total, value) => total + value.total, 0); +function sum(values: VisitTotals[]): VisitTotals { + return values.reduce( + (total, value) => ({ + validVisits: total.validVisits + value.validVisits, + invalidVisits: total.invalidVisits + value.invalidVisits + }), + { validVisits: 0, invalidVisits: 0 } + ); } -test('one refresh publishes all scopes and chart resolutions, excludes/deletes expired data and is repeatable', async () => { +test('publishes summary/detail pairs for all scopes, deletes expired history and is repeatable', async () => { const ctx = await context(); try { const now = Date.parse('2026-05-31T12:00:00Z'); @@ -52,18 +70,27 @@ test('one refresh publishes all scopes and chart resolutions, excludes/deletes e await insertVisit(ctx.db, { ...visit, visited_at: cutoff }); await insertVisit(ctx.db, { ...visit, path: '/', outcome: 'invalid', visited_at: now - 2000 }); await prepareVisitReports(ctx, now); - const reports = await readPreparedVisitReports(ctx.db, 'go.example.test', '/docs'); - expect(reports?.report.totals).toEqual({ total: 2, valid: 1, invalid: 1, base: 1 }); - expect(reports?.selection.totals.total).toBe(1); - expect(sum(reports!.report.daily)).toBe(2); - expect(sum(reports!.report.weekly)).toBe(2); - expect(sum(reports!.report.monthly)).toBe(2); - expect(reports?.report.browsers).toEqual([ - { name: 'Firefox', counts: { total: 2, valid: 1, invalid: 1, base: 1 } } + const summary = await readVisitSummary(ctx.db, domain); + const details = await readVisitDetails(ctx.db, domain); + expect(summary?.totals).toEqual({ validVisits: 1, invalidVisits: 1 }); + expect(summary?.scope).toEqual(domain); + expect(summary?.period.from).toBe('2026-02-28T12:00:00.000Z'); + expect(summary?.period.asOf).toBe('2026-05-31T12:00:00.000Z'); + expect(details?.scope).toEqual(summary?.scope); + expect(details?.period).toEqual(summary?.period); + expect((await readVisitSummary(ctx.db, docs))?.totals).toEqual({ + validVisits: 1, + invalidVisits: 0 + }); + expect(sum(details!.daily)).toEqual(summary?.totals); + expect(details?.browsers).toEqual([{ family: 'Firefox', validVisits: 1, invalidVisits: 1 }]); + expect(details?.operatingSystems).toEqual([ + { family: 'Linux', validVisits: 1, invalidVisits: 1 } ]); + expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toHaveLength(4); expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(2); await prepareVisitReports(ctx, now); - expect((await readPreparedVisitReports(ctx.db))?.report.totals.total).toBe(2); + expect((await readVisitSummary(ctx.db, overview))?.totals).toEqual(summary?.totals); } finally { await ctx.db.destroy(); } @@ -78,7 +105,31 @@ test('zero purges raw history and prepared output without leaving a longer-lived ctx.config.analytics.retention_months = 0; await prepareVisitReports(ctx, now); expect(await ctx.db.selectFrom('visits').select('id').execute()).toEqual([]); - expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); + } finally { + await ctx.db.destroy(); + } +}); + +test('encodes the whole batch before clearing output or pruning history', async () => { + const ctx = await context(); + try { + const now = Date.parse('2026-05-31T12:00:00Z'); + await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); + await prepareVisitReports(ctx, now); + const previous = await readVisitSummary(ctx.db, overview); + vi.mocked(pruneVisits).mockClear(); + const failure = new Error('Invalid report pair'); + vi.mocked(encodeVisitReports).mockClear(); + vi.mocked(encodeVisitReports).mockImplementationOnce(() => { + throw failure; + }); + + await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure); + expect(encodeVisitReports).toHaveBeenCalledOnce(); + expect(pruneVisits).not.toHaveBeenCalled(); + expect(await readVisitSummary(ctx.db, overview)).toEqual(previous); + expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(1); } finally { await ctx.db.destroy(); } @@ -89,16 +140,17 @@ test('clears old output before pruning and publishes nothing if pruning fails', try { const now = Date.parse('2026-05-31T12:00:00Z'); await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); - await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + await prepareVisitReports(ctx, now); + vi.mocked(pruneVisits).mockClear(); const failure = new Error('Pruning failed'); vi.mocked(pruneVisits).mockImplementationOnce(async () => { - expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); throw failure; }); await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure); expect(pruneVisits).toHaveBeenCalledExactlyOnceWith(ctx.db, Date.parse('2026-02-28T12:00:00Z')); - expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); } finally { await ctx.db.destroy(); } @@ -109,7 +161,8 @@ test('stops pruning and does not publish obsolete output when positive retention try { const now = Date.parse('2026-05-31T12:00:00Z'); await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); - await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + await prepareVisitReports(ctx, now); + vi.mocked(pruneVisits).mockClear(); vi.mocked(pruneVisits).mockImplementationOnce(async () => { ctx.config.analytics.retention_months = 9; return true; @@ -117,7 +170,7 @@ test('stops pruning and does not publish obsolete output when positive retention await prepareVisitReports(ctx, now); expect(pruneVisits).toHaveBeenCalledOnce(); - expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); } finally { await ctx.db.destroy(); } diff --git a/apps/worker/src/services/prepare-visit-reports.ts b/apps/worker/src/services/prepare-visit-reports.ts index 9e65462..700779b 100644 --- a/apps/worker/src/services/prepare-visit-reports.ts +++ b/apps/worker/src/services/prepare-visit-reports.ts @@ -1,4 +1,5 @@ import { setImmediate } from 'node:timers/promises'; +import { createVisitPeriod, encodeVisitReports } from '@goto/analytics'; import { pruneVisits, purgeVisits } from '@goto/db'; import { aggregateVisitReports, @@ -7,7 +8,6 @@ import { type ReportInventory } from '@goto/db/visit-reports'; import type { WorkerContext } from './context.js'; -import { reportPeriod } from './report-buckets.js'; import { buildVisitReports } from './build-visit-reports.js'; export async function prepareVisitReports( @@ -27,18 +27,20 @@ export async function prepareVisitReports( domain, paths: Object.keys(prepared.links) })); - const period = reportPeriod(months, asOf); - const counts = await aggregateVisitReports(context.db, period.since, asOf, inventory); + const period = createVisitPeriod(months, asOf); + const since = Date.parse(period.from); + const counts = await aggregateVisitReports(context.db, since, asOf, inventory); const reports = buildVisitReports(counts, inventory, period); + const rows = encodeVisitReports(reports); if (context.config.analytics.retention_months !== months) return; // Retire old output before deleting any history represented by it. await clearVisitReports(context.db); while (context.config.analytics.retention_months === months) { - const more = await pruneVisits(context.db, period.since); + const more = await pruneVisits(context.db, since); if (!more) break; await setImmediate(); } if (context.config.analytics.retention_months !== months) return; - await publishVisitReports(context.db, period, reports); + await publishVisitReports(context.db, rows); } diff --git a/apps/worker/src/services/report-buckets.test.ts b/apps/worker/src/services/report-buckets.test.ts deleted file mode 100644 index 7d6932b..0000000 --- a/apps/worker/src/services/report-buckets.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { expect, test } from 'vitest'; -import { reportPeriod } from './report-buckets.js'; - -test('prepares UTC days, Monday weeks and calendar months with honest partial edges', () => { - const asOf = Date.parse('2026-05-31T12:34:56.789Z'); - const period = reportPeriod(3, asOf); - expect(period.since).toBe(Date.parse('2026-02-28T12:34:56.789Z')); - expect(period.asOf).toBe(asOf); - expect(period.days[0]).toEqual({ - start: Date.parse('2026-02-28T00:00:00Z'), - end: Date.parse('2026-03-01T00:00:00Z'), - partial: true - }); - expect(period.days[1]).toEqual({ - start: Date.parse('2026-03-01T00:00:00Z'), - end: Date.parse('2026-03-02T00:00:00Z'), - partial: false - }); - expect(period.weeks[1]).toEqual({ - start: Date.parse('2026-03-02T00:00:00Z'), - end: Date.parse('2026-03-09T00:00:00Z'), - partial: false - }); - expect(period.months[1]).toEqual({ - start: Date.parse('2026-03-01T00:00:00Z'), - end: Date.parse('2026-04-01T00:00:00Z'), - partial: false - }); - expect(new Date(period.weeks[0]!.start).getUTCDay()).toBe(1); - expect(period.months[0]!.start).toBe(Date.parse('2026-02-01T00:00:00Z')); - expect(period.days.at(-1)!.partial).toBe(true); - expect(period.months.at(-1)!.partial).toBe(true); -}); diff --git a/apps/worker/src/services/report-buckets.ts b/apps/worker/src/services/report-buckets.ts deleted file mode 100644 index edfdbf2..0000000 --- a/apps/worker/src/services/report-buckets.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { TimeBucket, VisitReportPeriod } from '@goto/db/visit-reports'; -import { retentionCutoff, type RetentionMonths } from './retention.js'; - -type BucketSize = 'day' | 'week' | 'month'; - -export function reportPeriod(months: Exclude, asOf: number): VisitReportPeriod { - const since = retentionCutoff(months, asOf); - return { - timezone: 'UTC', - retentionMonths: months, - since, - asOf, - days: timeBuckets(since, asOf, 'day'), - weeks: timeBuckets(since, asOf, 'week'), - months: timeBuckets(since, asOf, 'month') - }; -} - -function timeBuckets(since: number, asOf: number, size: BucketSize): TimeBucket[] { - const date = new Date(since); - date.setUTCHours(0, 0, 0, 0); - if (size === 'week') { - const daysSinceMonday = (date.getUTCDay() + 6) % 7; - date.setUTCDate(date.getUTCDate() - daysSinceMonday); - } - if (size === 'month') date.setUTCDate(1); - const buckets: TimeBucket[] = []; - while (date.getTime() < asOf) { - const start = date.getTime(); - if (size === 'month') date.setUTCMonth(date.getUTCMonth() + 1); - else date.setUTCDate(date.getUTCDate() + (size === 'week' ? 7 : 1)); - const end = date.getTime(); - buckets.push({ start, end, partial: start < since || end > asOf }); - } - return buckets; -} diff --git a/apps/worker/src/services/report-schedule.test.ts b/apps/worker/src/services/report-schedule.test.ts index 25cdbdd..fc25a4b 100644 --- a/apps/worker/src/services/report-schedule.test.ts +++ b/apps/worker/src/services/report-schedule.test.ts @@ -1,8 +1,11 @@ +import { createVisitPeriod, encodeVisitReports } from '@goto/analytics'; import { validateConfig } from '@goto/config'; import { openDb, runMigrations } from '@goto/db'; +import { publishVisitReports } from '@goto/db/visit-reports'; import type { OpenWorkflow } from 'openworkflow'; import { expect, test, vi } from 'vitest'; import type { WorkerContext } from './context.js'; +import { buildVisitReports } from './build-visit-reports.js'; import { configureReportSchedule, requestReportRefresh } from './report-schedule.js'; import { refreshVisitReports, reportSchedule } from '../workflows/specs.js'; @@ -46,7 +49,13 @@ test('zero pauses hourly scheduling and purges prepared output even if raw histo expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(reportSchedule.id); await requestReportRefresh(ctx, sdk); expect(engine.runWorkflow).not.toHaveBeenCalled(); - await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + const period = createVisitPeriod(3, Date.now()); + const reports = buildVisitReports( + { daily: [], browsers: [], operatingSystems: [] }, + [], + period + ); + await publishVisitReports(ctx.db, encodeVisitReports(reports)); await requestReportRefresh(ctx, sdk); expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(refreshVisitReports, {}); } finally { diff --git a/apps/worker/src/services/report-schedule.ts b/apps/worker/src/services/report-schedule.ts index 709c591..482bc5c 100644 --- a/apps/worker/src/services/report-schedule.ts +++ b/apps/worker/src/services/report-schedule.ts @@ -19,8 +19,8 @@ export async function requestReportRefresh( if (context.config.analytics.retention_months === 0) { const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst(); const output = await context.db - .selectFrom('visit_reports') - .select('key') + .selectFrom('analytics_reports') + .select('id') .limit(1) .executeTakeFirst(); if (!visit && !output) return; diff --git a/apps/worker/src/services/retention.test.ts b/apps/worker/src/services/retention.test.ts deleted file mode 100644 index 2a04701..0000000 --- a/apps/worker/src/services/retention.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { expect, test } from 'vitest'; -import { retentionCutoff } from './retention.js'; - -test('uses UTC calendar months, clamping the day while preserving the time', () => { - const now = Date.parse('2026-05-31T12:34:56.789Z'); - expect(new Date(retentionCutoff(3, now)).toISOString()).toBe('2026-02-28T12:34:56.789Z'); - expect(new Date(retentionCutoff(9, now)).toISOString()).toBe('2025-08-31T12:34:56.789Z'); - expect(new Date(retentionCutoff(12, now)).toISOString()).toBe('2025-05-31T12:34:56.789Z'); -}); - -test('handles leap-day boundaries without overflowing into March', () => { - expect(new Date(retentionCutoff(3, Date.parse('2024-05-31T01:00:00Z'))).toISOString()).toBe( - '2024-02-29T01:00:00.000Z' - ); - expect(new Date(retentionCutoff(12, Date.parse('2024-02-29T01:00:00Z'))).toISOString()).toBe( - '2023-02-28T01:00:00.000Z' - ); -}); diff --git a/apps/worker/src/services/retention.ts b/apps/worker/src/services/retention.ts deleted file mode 100644 index 4a52a19..0000000 --- a/apps/worker/src/services/retention.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { GotoConfiguration } from '@goto/config'; - -export type RetentionMonths = GotoConfiguration['analytics']['retention_months']; - -export function retentionCutoff(months: Exclude, now = Date.now()): number { - const cutoff = new Date(now); - const day = cutoff.getUTCDate(); - cutoff.setUTCDate(1); - cutoff.setUTCMonth(cutoff.getUTCMonth() - months); - const lastDay = new Date( - Date.UTC(cutoff.getUTCFullYear(), cutoff.getUTCMonth() + 1, 0) - ).getUTCDate(); - cutoff.setUTCDate(Math.min(day, lastDay)); - return cutoff.getTime(); -} diff --git a/apps/worker/src/services/startup.test.ts b/apps/worker/src/services/startup.test.ts index 663e5d8..31d75a0 100644 --- a/apps/worker/src/services/startup.test.ts +++ b/apps/worker/src/services/startup.test.ts @@ -3,10 +3,13 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { createVisitPeriod, encodeVisitReports } from '@goto/analytics'; import { insertVisit, openDb, runMigrations } from '@goto/db'; +import { publishVisitReports } from '@goto/db/visit-reports'; import { Worker } from 'openworkflow'; import { afterEach, beforeEach, expect, test, vi } from 'vitest'; import { startWorkerService } from './startup.js'; +import { buildVisitReports } from './build-visit-reports.js'; let directory: string; @@ -61,12 +64,19 @@ test('starts an idle zero-policy service, migrates visit storage, and exposes he } }); -test('an existing zero-policy database is purged by a real startup workflow', async () => { +test('a real zero-policy startup workflow purges raw history and prepared output', async () => { await writeConfiguration(); const path = join(directory, 'data/visits.sqlite'); const db = await openDb(path); try { await runMigrations(db); + const period = createVisitPeriod(3, Date.now() - 1000); + const reports = buildVisitReports( + { daily: [], browsers: [], operatingSystems: [] }, + [], + period + ); + await publishVisitReports(db, encodeVisitReports(reports)); await insertVisit(db, { domain: 'go.example.test', path: '/docs', @@ -86,6 +96,7 @@ test('an existing zero-policy database is purged by a real startup workflow', as const reader = await openDb(path, { readOnly: true }); try { expect(await reader.selectFrom('visits').selectAll().execute()).toEqual([]); + expect(await reader.selectFrom('analytics_reports').select('id').execute()).toEqual([]); } finally { await reader.destroy(); } diff --git a/packages/db/package.json b/packages/db/package.json index 47df02a..09b2a10 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -28,6 +28,7 @@ "test:coverage": "vitest run --coverage" }, "dependencies": { + "@goto/analytics": "workspace:*", "kysely": "catalog:" }, "devDependencies": { diff --git a/packages/db/src/migrations/0007_analytics_reports.ts b/packages/db/src/migrations/0007_analytics_reports.ts new file mode 100644 index 0000000..7eebc71 --- /dev/null +++ b/packages/db/src/migrations/0007_analytics_reports.ts @@ -0,0 +1,41 @@ +import { sql, type Kysely, type Migration } from 'kysely'; + +export const analyticsReports: Migration = { + async up(db: Kysely): Promise { + await db.schema + .createTable('analytics_reports') + .addColumn('id', 'integer', (column) => column.primaryKey()) + .addColumn('scope_kind', 'text', (column) => column.notNull()) + .addColumn('domain', 'text') + .addColumn('path', 'text') + .addColumn('summary_json', 'text', (column) => column.notNull()) + .addColumn('details_json', 'text', (column) => column.notNull()) + .addCheckConstraint( + 'analytics_reports_scope', + sql` + (scope_kind = 'overview' AND domain IS NULL AND path IS NULL) OR + (scope_kind = 'domain' AND domain IS NOT NULL AND length(domain) > 0 AND path IS NULL) OR + (scope_kind = 'link' AND domain IS NOT NULL AND length(domain) > 0 + AND path IS NOT NULL AND substr(path, 1, 1) = '/') + ` + ) + .addCheckConstraint('analytics_reports_summary_json', sql`json_valid(summary_json)`) + .addCheckConstraint('analytics_reports_details_json', sql`json_valid(details_json)`) + .execute(); + await sql`CREATE UNIQUE INDEX analytics_reports_overview + ON analytics_reports (scope_kind) WHERE scope_kind = 'overview'`.execute(db); + await sql`CREATE UNIQUE INDEX analytics_reports_domain + ON analytics_reports (domain) WHERE scope_kind = 'domain'`.execute(db); + await sql`CREATE UNIQUE INDEX analytics_reports_link + ON analytics_reports (domain, path) WHERE scope_kind = 'link'`.execute(db); + await db.schema.dropTable('visit_reports').execute(); + }, + async down(db: Kysely): Promise { + await db.schema.dropTable('analytics_reports').execute(); + await db.schema + .createTable('visit_reports') + .addColumn('key', 'text', (column) => column.primaryKey()) + .addColumn('data', 'text', (column) => column.notNull()) + .execute(); + } +}; diff --git a/packages/db/src/migrations/index.ts b/packages/db/src/migrations/index.ts index 218f0ed..52720ce 100644 --- a/packages/db/src/migrations/index.ts +++ b/packages/db/src/migrations/index.ts @@ -4,6 +4,7 @@ import { visitRetention } from './0003_visit_retention.js'; import { removeVisitCollection } from './0004_remove_visit_collection.js'; import { addVisitOutcome } from './0005_visit_outcome.js'; import { visitReports } from './0006_visit_reports.js'; +import { analyticsReports } from './0007_analytics_reports.js'; export const migrations = { '0001_visits': initial, @@ -11,5 +12,6 @@ export const migrations = { '0003_visit_retention': visitRetention, '0004_remove_visit_collection': removeVisitCollection, '0005_visit_outcome': addVisitOutcome, - '0006_visit_reports': visitReports + '0006_visit_reports': visitReports, + '0007_analytics_reports': analyticsReports }; diff --git a/packages/db/src/storage.test.ts b/packages/db/src/storage.test.ts index 8141ab3..00a13e6 100644 --- a/packages/db/src/storage.test.ts +++ b/packages/db/src/storage.test.ts @@ -55,11 +55,17 @@ test('opens a real WAL database and applies versioned migrations once', async () '0003_visit_retention', '0004_remove_visit_collection', '0005_visit_outcome', - '0006_visit_reports' + '0006_visit_reports', + '0007_analytics_reports' ]); const tables = await db.introspection.getTables(); const names = tables.map((table) => table.name); expect(names).not.toContain('visit_collection'); + expect(names).not.toContain('visit_reports'); + expect(names).toContain('analytics_reports'); + expect( + tables.find((table) => table.name === 'analytics_reports')?.columns.map(({ name }) => name) + ).toEqual(['id', 'scope_kind', 'domain', 'path', 'summary_json', 'details_json']); expect(await runMigrations(db)).toEqual([]); } finally { await db.destroy(); @@ -109,7 +115,7 @@ test('persists request-time outcomes for the same logical path on different doma } }); -test('adds request-time labels without changing existing visit data', async () => { +test('upgrades legacy visits and discards only the obsolete report cache', async () => { const db = await openDb(join(directory, 'legacy.sqlite')); try { const migrator = new Migrator({ @@ -127,13 +133,14 @@ test('adds request-time labels without changing existing visit data', async () = VALUES (${visit.domain}, ${visit.path}, ${visit.visited_at}, ${visit.ip}, ${visit.user_agent}, ${visit.accept_language}) `.execute(db); - expect(await runMigrations(db)).toEqual([ - '0002_visit_classification', - '0003_visit_retention', - '0004_remove_visit_collection', - '0005_visit_outcome', - '0006_visit_reports' - ]); + const upgrade = await migrator.migrateTo('0006_visit_reports'); + if (upgrade.error) throw upgrade.error; + await sql`INSERT INTO visit_reports (key, data) VALUES ('overview', 'obsolete cache')`.execute( + db + ); + + expect(await runMigrations(db)).toEqual(['0007_analytics_reports']); + expect(await db.selectFrom('analytics_reports').selectAll().execute()).toEqual([]); expect(await db.selectFrom('visits').selectAll().executeTakeFirstOrThrow()).toEqual({ ...visit, id: 1, @@ -171,7 +178,8 @@ test('a failed migration rolls back schema and bookkeeping so startup can retry' '0003_visit_retention', '0004_remove_visit_collection', '0005_visit_outcome', - '0006_visit_reports' + '0006_visit_reports', + '0007_analytics_reports' ]); expect(await runMigrations(db)).toEqual([]); } finally { diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 0af8fc5..82c2b30 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -15,14 +15,18 @@ export interface VisitTable { os: string; } -export interface VisitReportTable { - key: string; - data: string; +export interface AnalyticsReportTable { + id: Generated; + scope_kind: 'overview' | 'domain' | 'link'; + domain: string | null; + path: string | null; + summary_json: string; + details_json: string; } export interface Database { visits: VisitTable; - visit_reports: VisitReportTable; + analytics_reports: AnalyticsReportTable; } export type ClassifiedVisit = Omit, 'id'>; diff --git a/packages/db/src/visit-reports/index.ts b/packages/db/src/visit-reports/index.ts index 5ee76d2..c4b496e 100644 --- a/packages/db/src/visit-reports/index.ts +++ b/packages/db/src/visit-reports/index.ts @@ -1,16 +1,13 @@ export { aggregateVisitReports } from './queries.js'; -export { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js'; -export { visitReportKey, zeroVisitCounts } from './types.js'; +export { + clearVisitReports, + publishVisitReports, + readVisitSummary, + readVisitDetails +} from './storage.js'; export type { AggregatedVisitReports, DailyVisitCounts, - FamilyVisitCounts, - PreparedVisitReport, ReportInventory, - ScopedFamilyCounts, - SerializedVisitReport, - StoredVisitReports, - TimeBucket, - VisitCounts, - VisitReportPeriod + ScopedFamilyCounts } from './types.js'; diff --git a/packages/db/src/visit-reports/queries.test.ts b/packages/db/src/visit-reports/queries.test.ts index b6f52fc..edee7fb 100644 --- a/packages/db/src/visit-reports/queries.test.ts +++ b/packages/db/src/visit-reports/queries.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, expect, test } from 'vitest'; import { openDb, runMigrations, type ClassifiedVisit } from '../index.js'; import { aggregateVisitReports } from './queries.js'; -import { visitReportKey, type ReportInventory } from './types.js'; +import type { ReportInventory } from './types.js'; let db: Awaited>; @@ -71,75 +71,228 @@ async function seedVisits(): Promise { .execute(); } -test('aggregates UTC days and stored families for overview, current domains and known paths', async () => { +test('aggregates explicit UTC scope identities, outcomes and root subsets', async () => { await seedVisits(); const result = await aggregateVisitReports(db, since, asOf, inventory); - const overview = visitReportKey(); - const domainScope = visitReportKey(domain); - const docs = visitReportKey(domain, '/docs'); - const root = visitReportKey(domain, '/'); - const otherDomainScope = visitReportKey(otherDomain); - const otherDocs = visitReportKey(otherDomain, '/docs'); expect(result.daily).toHaveLength(8); expect(result.daily).toEqual( expect.arrayContaining([ - { scope: overview, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, - { scope: overview, day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 }, - { scope: domainScope, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, - { scope: domainScope, day: '2026-02-01', total: 3, valid: 1, invalid: 2, base: 2 }, - { scope: docs, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, - { scope: root, day: '2026-02-01', total: 2, valid: 1, invalid: 1, base: 2 }, - { scope: otherDomainScope, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 }, - { scope: otherDocs, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 } + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-01-31', + validVisits: 1, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-02-01', + validVisits: 1, + invalidVisits: 4, + baseValidVisits: 1, + baseInvalidVisits: 2 + }, + { + scopeKind: 'domain', + domain, + path: null, + day: '2026-01-31', + validVisits: 1, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + scopeKind: 'domain', + domain, + path: null, + day: '2026-02-01', + validVisits: 1, + invalidVisits: 2, + baseValidVisits: 1, + baseInvalidVisits: 1 + }, + { + scopeKind: 'link', + domain, + path: '/docs', + day: '2026-01-31', + validVisits: 1, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + scopeKind: 'link', + domain, + path: '/', + day: '2026-02-01', + validVisits: 1, + invalidVisits: 1, + baseValidVisits: 1, + baseInvalidVisits: 1 + }, + { + scopeKind: 'domain', + domain: otherDomain, + path: null, + day: '2026-02-01', + validVisits: 0, + invalidVisits: 1, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + scopeKind: 'link', + domain: otherDomain, + path: '/docs', + day: '2026-02-01', + validVisits: 0, + invalidVisits: 1, + baseValidVisits: 0, + baseInvalidVisits: 0 + } ]) ); - expect(result.browsers.filter(({ scope }) => scope === overview)).toEqual([ - { scope: overview, name: 'Firefox', total: 4, valid: 1, invalid: 3, base: 2 }, - { scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, - { scope: overview, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + expect(result.browsers.filter(({ scopeKind }) => scopeKind === 'overview')).toEqual([ + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Firefox', + validVisits: 1, + invalidVisits: 3 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Other', + validVisits: 0, + invalidVisits: 1 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Safari', + validVisits: 1, + invalidVisits: 0 + } ]); - expect(result.operatingSystems.filter(({ scope }) => scope === overview)).toEqual([ - { scope: overview, name: 'Linux', total: 3, valid: 1, invalid: 2, base: 2 }, - { scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, - { scope: overview, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 }, - { scope: overview, name: 'macOS', total: 1, valid: 1, invalid: 0, base: 1 } + expect(result.operatingSystems.filter(({ scopeKind }) => scopeKind === 'overview')).toEqual([ + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Linux', + validVisits: 1, + invalidVisits: 2 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Other', + validVisits: 0, + invalidVisits: 1 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'Windows', + validVisits: 0, + invalidVisits: 1 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + family: 'macOS', + validVisits: 1, + invalidVisits: 0 + } ]); - expect(result.browsers.filter(({ scope }) => scope === domainScope)).toEqual([ - { scope: domainScope, name: 'Firefox', total: 2, valid: 1, invalid: 1, base: 1 }, - { scope: domainScope, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, - { scope: domainScope, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + expect( + result.browsers.filter(({ scopeKind, path }) => scopeKind === 'link' && path === '/') + ).toEqual([ + { scopeKind: 'link', domain, path: '/', family: 'Firefox', validVisits: 0, invalidVisits: 1 }, + { scopeKind: 'link', domain, path: '/', family: 'Safari', validVisits: 1, invalidVisits: 0 } ]); - expect(result.browsers.filter(({ scope }) => scope === root)).toEqual([ - { scope: root, name: 'Firefox', total: 1, valid: 0, invalid: 1, base: 1 }, - { scope: root, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + expect( + result.operatingSystems.filter( + ({ scopeKind, domain: rowDomain }) => scopeKind === 'link' && rowDomain === otherDomain + ) + ).toEqual([ + { + scopeKind: 'link', + domain: otherDomain, + path: '/docs', + family: 'Windows', + validVisits: 0, + invalidVisits: 1 + } ]); - expect(result.operatingSystems.filter(({ scope }) => scope === otherDocs)).toEqual([ - { scope: otherDocs, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 } - ]); - const familyScopes = new Set( - [...result.browsers, ...result.operatingSystems].map(({ scope }) => scope) - ); - expect(familyScopes).toEqual( - new Set([overview, domainScope, docs, root, otherDomainScope, otherDocs]) - ); }); -test('empty inventory still aggregates historical visits only under overview', async () => { +test('empty inventory keeps historical overview traffic but no domain or link scopes', async () => { await seedVisits(); const result = await aggregateVisitReports(db, since, asOf, []); expect(result.daily).toEqual([ - { scope: 'overview', day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, - { scope: 'overview', day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 } - ]); - expect(result.browsers.map(({ scope }) => scope)).toEqual(['overview', 'overview', 'overview']); - expect(result.operatingSystems.map(({ scope }) => scope)).toEqual([ - 'overview', - 'overview', - 'overview', - 'overview' + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-01-31', + validVisits: 1, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }, + { + scopeKind: 'overview', + domain: null, + path: null, + day: '2026-02-01', + validVisits: 1, + invalidVisits: 4, + baseValidVisits: 1, + baseInvalidVisits: 2 + } ]); + expect( + [...result.browsers, ...result.operatingSystems].every( + (row) => row.scopeKind === 'overview' && row.domain === null && row.path === null + ) + ).toBe(true); +}); + +test('paths remain exact bound identities, including punctuation and Unicode', async () => { + const path = '/café,"docs'; + await db + .insertInto('visits') + .values({ ...visit, path }) + .execute(); + const result = await aggregateVisitReports(db, since, asOf, [{ domain, paths: [path] }]); + + expect(result.daily.find((row) => row.scopeKind === 'link')).toEqual({ + scopeKind: 'link', + domain, + path, + day: '2026-01-31', + validVisits: 1, + invalidVisits: 0, + baseValidVisits: 0, + baseInvalidVisits: 0 + }); }); test('no visits produces no aggregate rows for configured scopes', async () => { diff --git a/packages/db/src/visit-reports/queries.ts b/packages/db/src/visit-reports/queries.ts index 75be418..93895dd 100644 --- a/packages/db/src/visit-reports/queries.ts +++ b/packages/db/src/visit-reports/queries.ts @@ -1,11 +1,10 @@ import { sql, type Kysely } from 'kysely'; import type { Database } from '../types.js'; -import { - visitReportKey, - type AggregatedVisitReports, - type DailyVisitCounts, - type ReportInventory, - type ScopedFamilyCounts +import type { + AggregatedVisitReports, + DailyVisitCounts, + ReportInventory, + ScopedFamilyCounts } from './types.js'; export async function aggregateVisitReports( @@ -15,31 +14,36 @@ export async function aggregateVisitReports( inventory: ReportInventory ): Promise { const scopes = scopedVisits(since, asOf, inventory); - const counts = visitCountColumns(); + const identity = sql`scope_kind AS "scopeKind", domain, path`; + const counts = sql` + SUM(CASE WHEN outcome = 'valid' THEN 1 ELSE 0 END) AS "validVisits", + SUM(CASE WHEN outcome = 'invalid' THEN 1 ELSE 0 END) AS "invalidVisits" + `; return db.transaction().execute(async (transaction) => { const daily = await sql` ${scopes} - SELECT scope, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts} + SELECT ${identity}, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts}, + SUM(CASE WHEN visit_path = '/' AND outcome = 'valid' THEN 1 ELSE 0 END) AS "baseValidVisits", + SUM(CASE WHEN visit_path = '/' AND outcome = 'invalid' THEN 1 ELSE 0 END) AS "baseInvalidVisits" FROM scoped_visits - GROUP BY scope, day - ORDER BY scope, day + GROUP BY scope_kind, domain, path, day + ORDER BY scope_kind, domain, path, day `.execute(transaction); const browsers = await sql` ${scopes} - SELECT scope, browser AS name, ${counts} + SELECT ${identity}, browser AS family, ${counts} FROM scoped_visits - GROUP BY scope, browser - ORDER BY scope, name + GROUP BY scope_kind, domain, path, browser + ORDER BY scope_kind, domain, path, family `.execute(transaction); const operatingSystems = await sql` ${scopes} - SELECT scope, os AS name, ${counts} + SELECT ${identity}, os AS family, ${counts} FROM scoped_visits - GROUP BY scope, os - ORDER BY scope, name + GROUP BY scope_kind, domain, path, os + ORDER BY scope_kind, domain, path, family `.execute(transaction); - return { daily: daily.rows, browsers: browsers.rows, @@ -56,32 +60,25 @@ function scopedVisits(since: number, asOf: number, inventory: ReportInventory) { return sql`(domain = ${domain} AND path IN (${sql.join(currentPaths)}))`; }); const currentLink = linkRules.length === 0 ? sql`0` : sql.join(linkRules, sql` OR `); - return sql` WITH retained_visits AS ( SELECT domain, path, outcome, visited_at, browser, os FROM visits WHERE visited_at >= ${since} AND visited_at < ${asOf} ), scoped_visits AS ( - SELECT ${visitReportKey()} AS scope, path, outcome, visited_at, browser, os + SELECT 'overview' AS scope_kind, NULL AS domain, NULL AS path, + path AS visit_path, outcome, visited_at, browser, os FROM retained_visits UNION ALL - SELECT json_array(domain) AS scope, path, outcome, visited_at, browser, os + SELECT 'domain' AS scope_kind, domain, NULL AS path, + path AS visit_path, outcome, visited_at, browser, os FROM retained_visits WHERE ${currentDomain} UNION ALL - SELECT json_array(domain, path) AS scope, path, outcome, visited_at, browser, os + SELECT 'link' AS scope_kind, domain, path, + path AS visit_path, outcome, visited_at, browser, os FROM retained_visits WHERE ${currentLink} ) `; } - -function visitCountColumns() { - return sql` - COUNT(*) AS total, - SUM(CASE WHEN outcome = ${'valid'} THEN 1 ELSE 0 END) AS valid, - SUM(CASE WHEN outcome = ${'invalid'} THEN 1 ELSE 0 END) AS invalid, - SUM(CASE WHEN path = ${'/'} THEN 1 ELSE 0 END) AS base - `; -} diff --git a/packages/db/src/visit-reports/storage.test.ts b/packages/db/src/visit-reports/storage.test.ts index e6b7f16..472831d 100644 --- a/packages/db/src/visit-reports/storage.test.ts +++ b/packages/db/src/visit-reports/storage.test.ts @@ -1,27 +1,30 @@ -import { afterEach, beforeEach, expect, test } from 'vitest'; -import { openDb, runMigrations } from '../index.js'; -import { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js'; import { - visitReportKey, - zeroVisitCounts, + createVisitPeriod, + encodeVisitReports, + visitDays, + zeroVisitTotals, type PreparedVisitReport, - type SerializedVisitReport, - type VisitCounts, - type VisitReportPeriod -} from './types.js'; + type ReportScope, + type VisitSummary, + type VisitTotals +} from '@goto/analytics'; +import { DatabaseSync } from 'node:sqlite'; +import { sql } from 'kysely'; +import { afterEach, beforeEach, expect, test, vi } from 'vitest'; +import { openDb, runMigrations } from '../index.js'; +import { + clearVisitReports, + publishVisitReports, + readVisitDetails, + readVisitSummary +} from './storage.js'; let db: Awaited>; - const domain = 'go.example.test'; -const period: VisitReportPeriod = { - timezone: 'UTC', - retentionMonths: 3, - since: Date.UTC(2026, 0, 1), - asOf: Date.UTC(2026, 0, 2), - days: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 0, 2), partial: false }], - weeks: [{ start: Date.UTC(2025, 11, 29), end: Date.UTC(2026, 0, 5), partial: true }], - months: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 1, 1), partial: true }] -}; +const overviewScope = { kind: 'overview' } as const; +const domainScope = { kind: 'domain', domain } as const; +const rootScope = { kind: 'link', domain, path: '/' } as const; +const asOf = Date.UTC(2026, 3, 1); beforeEach(async () => { db = await openDb(':memory:'); @@ -32,132 +35,195 @@ afterEach(async () => { await db.destroy(); }); -function preparedReport(counts: VisitCounts = zeroVisitCounts()): PreparedVisitReport { +function preparedReport( + scope: ReportScope, + cutoff = asOf, + totals: VisitTotals = zeroVisitTotals() +): PreparedVisitReport { + const period = createVisitPeriod(3, cutoff); + let summary: VisitSummary; + if (scope.kind === 'overview') { + summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), domains: [] }; + } else if (scope.kind === 'domain') { + summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), links: [] }; + } else { + summary = { version: 1, scope, period, totals }; + } + const populated = totals.validVisits + totals.invalidVisits > 0; return { - totals: counts, - daily: [counts], - weekly: [counts], - monthly: [counts], - browsers: [], - operatingSystems: [], - domainCounts: [], - linkCounts: [] + summary, + details: { + version: 1, + scope, + period, + daily: visitDays(period).map((day, index) => ({ + ...day, + ...(index === 0 ? totals : zeroVisitTotals()) + })), + browsers: populated ? [{ family: 'Firefox', ...totals }] : [], + operatingSystems: populated ? [{ family: 'Linux', ...totals }] : [] + } }; } -function serializedReport(key: string, report: PreparedVisitReport): SerializedVisitReport { - return { key, data: JSON.stringify(report) }; -} +test('reads summary and details for each explicit SQL scope including root', async () => { + const reports = [ + preparedReport(overviewScope, asOf, { validVisits: 2, invalidVisits: 1 }), + preparedReport(domainScope, asOf, { validVisits: 1, invalidVisits: 1 }), + preparedReport(rootScope) + ]; + await publishVisitReports(db, encodeVisitReports(reports)); -test('reads the shared period, scope report and selected path from prepared rows', async () => { - const selection = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); - selection.browsers = [{ name: 'Firefox', counts: selection.totals }]; - selection.operatingSystems = [{ name: 'Linux', counts: selection.totals }]; - const domainReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 1 }); - domainReport.linkCounts = [{ path: '/docs', counts: selection.totals }]; - const overview = preparedReport({ total: 3, valid: 2, invalid: 1, base: 1 }); - overview.domainCounts = [{ domain, counts: domainReport.totals }]; - await publishVisitReports(db, period, [ - serializedReport('empty', preparedReport()), - serializedReport(visitReportKey(), overview), - serializedReport(visitReportKey(domain), domainReport), - serializedReport(visitReportKey(domain, '/docs'), selection) + for (const report of reports) { + expect(await readVisitSummary(db, report.summary.scope)).toEqual(report.summary); + expect(await readVisitDetails(db, report.summary.scope)).toEqual(report.details); + } + expect( + await db + .selectFrom('analytics_reports') + .select(['scope_kind', 'domain', 'path']) + .orderBy('id') + .execute() + ).toEqual([ + { scope_kind: 'overview', domain: null, path: null }, + { scope_kind: 'domain', domain, path: null }, + { scope_kind: 'link', domain, path: '/' } ]); - - expect(await readPreparedVisitReports(db)).toEqual({ - period, - report: overview, - selection: overview - }); - expect(await readPreparedVisitReports(db, domain)).toEqual({ - period, - report: domainReport, - selection: domainReport - }); - expect(await readPreparedVisitReports(db, domain, '/docs')).toEqual({ - period, - report: domainReport, - selection - }); }); -test('publication replaces all output rows and a failed replacement preserves the previous set', async () => { - const oldReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); - await publishVisitReports(db, period, [ - serializedReport(visitReportKey(), oldReport), - serializedReport(visitReportKey(domain), oldReport) - ]); - const replacementPeriod = { ...period, retentionMonths: 9 as const }; - const replacementReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 0 }); - const replacementRows = [serializedReport(visitReportKey(), replacementReport)]; - await publishVisitReports(db, replacementPeriod, replacementRows); +test('publication replaces the complete batch, including when the replacement is empty', async () => { + await publishVisitReports( + db, + encodeVisitReports([preparedReport(overviewScope), preparedReport(domainScope)]) + ); + const replacement = preparedReport(overviewScope, asOf + 86_400_000); + await publishVisitReports(db, encodeVisitReports([replacement])); - expect(await db.selectFrom('visit_reports').select('key').orderBy('key').execute()).toEqual([ - { key: 'overview' }, - { key: 'period' } - ]); - expect(await readPreparedVisitReports(db, domain)).toBeNull(); - await expect( - publishVisitReports(db, period, [ - serializedReport(visitReportKey(), oldReport), - serializedReport(visitReportKey(), oldReport) - ]) - ).rejects.toThrow(); - expect(await readPreparedVisitReports(db)).toEqual({ - period: replacementPeriod, - report: replacementReport, - selection: replacementReport - }); + expect(await readVisitSummary(db, overviewScope)).toEqual(replacement.summary); + expect(await readVisitSummary(db, domainScope)).toBeNull(); + await publishVisitReports(db, []); + expect(await db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); }); -test('a published zero-count path uses the empty template but a missing domain stays unavailable', async () => { - const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 1 }); - domainReport.linkCounts = [{ path: '/new', counts: zeroVisitCounts() }]; - const empty = preparedReport(); - await publishVisitReports(db, period, [ - serializedReport(visitReportKey(domain), domainReport), - serializedReport('empty', empty) - ]); +test('native uniqueness and JSON failures roll back publication deletion', async () => { + const original = preparedReport(overviewScope); + await publishVisitReports(db, encodeVisitReports([original])); + const [replacement] = encodeVisitReports([preparedReport(domainScope)]); + if (!replacement) throw new Error('Expected an encoded report fixture'); - expect(await readPreparedVisitReports(db, domain, '/new')).toEqual({ - period, - report: domainReport, - selection: empty - }); - expect(await readPreparedVisitReports(db, 'new.example.test', '/new')).toBeNull(); + for (const batch of [ + [replacement, replacement], + [{ ...replacement, summaryJson: '{' }], + [{ ...replacement, detailsJson: '{' }] + ]) { + await expect(publishVisitReports(db, batch)).rejects.toThrow(); + expect(await readVisitSummary(db, overviewScope)).toEqual(original.summary); + expect(await readVisitDetails(db, overviewScope)).toEqual(original.details); + expect(await readVisitSummary(db, domainScope)).toBeNull(); + } }); -test('missing link bodies stay unavailable for unpublished paths and populated links', async () => { - const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); - domainReport.linkCounts = [{ path: '/docs', counts: domainReport.totals }]; - await publishVisitReports(db, period, [ - serializedReport(visitReportKey(domain), domainReport), - serializedReport('empty', preparedReport()) - ]); +test('missing scopes are null, not implicit empty reports or parent fallbacks', async () => { + const report = preparedReport(domainScope); + if (!('links' in report.summary)) throw new Error('Expected a domain fixture'); + report.summary.links = [{ path: '/new', totals: zeroVisitTotals() }]; + await publishVisitReports(db, encodeVisitReports([report])); - expect(await readPreparedVisitReports(db, domain, '/new')).toBeNull(); - expect(await readPreparedVisitReports(db, domain, '/docs')).toBeNull(); -}); - -test('missing metadata is unavailable and clearing deletes every output row', async () => { - expect(await readPreparedVisitReports(db)).toBeNull(); - await db - .insertInto('visit_reports') - .values(serializedReport(visitReportKey(), preparedReport())) - .execute(); - expect(await readPreparedVisitReports(db)).toBeNull(); - - await publishVisitReports(db, period, [ - serializedReport(visitReportKey(), preparedReport()), - serializedReport('empty', preparedReport()) - ]); + expect(await readVisitSummary(db, overviewScope)).toBeNull(); + expect(await readVisitSummary(db, { kind: 'link', domain, path: '/new' })).toBeNull(); + expect(await readVisitDetails(db, rootScope)).toBeNull(); await clearVisitReports(db); - expect(await db.selectFrom('visit_reports').selectAll().execute()).toEqual([]); - expect(await readPreparedVisitReports(db)).toBeNull(); + expect(await readVisitSummary(db, domainScope)).toBeNull(); + expect(await db.selectFrom('analytics_reports').select('id').execute()).toEqual([]); }); -test('malformed stored JSON is left for the caller to handle', async () => { - await publishVisitReports(db, period, [{ key: visitReportKey(), data: '{' }]); +test('summary and details readers select and decode only their own column', async () => { + const report = preparedReport(overviewScope); + const rows = encodeVisitReports([report]); + const [row] = rows; + if (!row) throw new Error('Expected an encoded report fixture'); + await publishVisitReports(db, rows); + await db.updateTable('analytics_reports').set({ details_json: '{}' }).execute(); - await expect(readPreparedVisitReports(db)).rejects.toThrow(SyntaxError); + const statements = vi.spyOn(DatabaseSync.prototype, 'prepare'); + try { + expect(await readVisitSummary(db, overviewScope)).toEqual(report.summary); + expect(statements.mock.calls.map(([query]) => query)).toEqual([ + expect.stringMatching(/^select\s+"summary_json"\s+from\s+/i) + ]); + statements.mockClear(); + await expect(readVisitDetails(db, overviewScope)).rejects.toThrow('Invalid analytics report'); + expect(statements.mock.calls.map(([query]) => query)).toEqual([ + expect.stringMatching(/^select\s+"details_json"\s+from\s+/i) + ]); + } finally { + statements.mockRestore(); + } + await db + .updateTable('analytics_reports') + .set({ summary_json: '{}', details_json: row.detailsJson }) + .execute(); + expect(await readVisitDetails(db, overviewScope)).toEqual(report.details); + await expect(readVisitSummary(db, overviewScope)).rejects.toThrow('Invalid analytics report'); +}); + +test('database constraints reject invalid scope shapes, duplicate identities and malformed JSON', async () => { + const reports = encodeVisitReports( + [overviewScope, domainScope, rootScope].map((scope) => preparedReport(scope)) + ); + await publishVisitReports(db, reports); + + for (const report of reports) { + const scope = report.scope; + await expect( + db + .insertInto('analytics_reports') + .values({ + scope_kind: scope.kind, + domain: scope.kind === 'overview' ? null : scope.domain, + path: scope.kind === 'link' ? scope.path : null, + summary_json: report.summaryJson, + details_json: report.detailsJson + }) + .execute() + ).rejects.toThrow(); + } + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('overview', 'bad.example.test', NULL, '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('domain', NULL, NULL, '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('link', 'go.example.test', NULL, '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('unknown', NULL, NULL, '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('domain', '', NULL, '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('link', '', '/docs', '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('link', 'go.example.test', '', '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json) + VALUES ('link', 'go.example.test', 'docs', '{}', '{}')`.execute(db) + ).rejects.toThrow(); + await expect( + db.updateTable('analytics_reports').set({ summary_json: '{' }).execute() + ).rejects.toThrow(); + await expect( + db.updateTable('analytics_reports').set({ details_json: '{' }).execute() + ).rejects.toThrow(); }); diff --git a/packages/db/src/visit-reports/storage.ts b/packages/db/src/visit-reports/storage.ts index 0f0b85b..2a81d6b 100644 --- a/packages/db/src/visit-reports/storage.ts +++ b/packages/db/src/visit-reports/storage.ts @@ -1,59 +1,78 @@ +import { + parseVisitDetails, + parseVisitSummary, + type ReportScope, + type SerializedVisitReport, + type VisitDetails, + type VisitSummary +} from '@goto/analytics'; import type { Kysely } from 'kysely'; import type { Database } from '../types.js'; -import { - visitReportKey, - type PreparedVisitReport, - type SerializedVisitReport, - type StoredVisitReports, - type VisitReportPeriod -} from './types.js'; export async function clearVisitReports(db: Kysely): Promise { - await db.deleteFrom('visit_reports').execute(); + await db.deleteFrom('analytics_reports').execute(); } export async function publishVisitReports( db: Kysely, - period: VisitReportPeriod, reports: SerializedVisitReport[] ): Promise { - const rows = [{ key: 'period', data: JSON.stringify(period) }, ...reports]; - + const rows = reports.map(({ scope, summaryJson, detailsJson }) => { + let domain: string | null = null; + let path: string | null = null; + if (scope.kind !== 'overview') domain = scope.domain; + if (scope.kind === 'link') path = scope.path; + return { + scope_kind: scope.kind, + domain, + path, + summary_json: summaryJson, + details_json: detailsJson + }; + }); await db.transaction().execute(async (transaction) => { - await transaction.deleteFrom('visit_reports').execute(); - await transaction.insertInto('visit_reports').values(rows).execute(); + await transaction.deleteFrom('analytics_reports').execute(); + if (rows.length > 0) await transaction.insertInto('analytics_reports').values(rows).execute(); }); } -export async function readPreparedVisitReports( +export async function readVisitSummary( db: Kysely, - domain?: string, - path?: string -): Promise { - const reportKey = visitReportKey(domain); - const selectionKey = visitReportKey(domain, path); - const keys = [...new Set(['period', reportKey, selectionKey, 'empty'])]; - const rows = await db.selectFrom('visit_reports').selectAll().where('key', 'in', keys).execute(); - const dataByKey = new Map(rows.map(({ key, data }) => [key, data])); - const periodData = dataByKey.get('period'); - const reportData = dataByKey.get(reportKey); - if (periodData === undefined || reportData === undefined) return null; + scope: ReportScope +): Promise { + const row = await reportScopeQuery(db, scope).select('summary_json').executeTakeFirst(); + return row ? parseVisitSummary(row.summary_json, scope) : null; +} - const report = JSON.parse(reportData) as PreparedVisitReport; - let selection = report; - if (selectionKey !== reportKey) { - let selectionData = dataByKey.get(selectionKey); - if (selectionData === undefined) { - const linkCounts = report.linkCounts.find((link) => link.path === path)?.counts; - if (linkCounts?.total !== 0) return null; - selectionData = dataByKey.get('empty'); - } - if (selectionData === undefined) return null; - selection = JSON.parse(selectionData) as PreparedVisitReport; +export async function readVisitDetails( + db: Kysely, + scope: ReportScope +): Promise { + const row = await reportScopeQuery(db, scope).select('details_json').executeTakeFirst(); + return row ? parseVisitDetails(row.details_json, scope) : null; +} + +function reportScopeQuery(db: Kysely, scope: ReportScope) { + if (scope.kind === 'overview') { + return db + .selectFrom('analytics_reports') + .where('scope_kind', '=', 'overview') + .where('domain', 'is', null) + .where('path', 'is', null); + } + if (scope.kind !== 'domain' && scope.kind !== 'link') { + throw new Error('Invalid analytics report'); + } + if (typeof scope.domain !== 'string' || scope.domain.length === 0) { + throw new Error('Invalid analytics report'); + } + const query = db + .selectFrom('analytics_reports') + .where('scope_kind', '=', scope.kind) + .where('domain', '=', scope.domain); + if (scope.kind === 'domain') return query.where('path', 'is', null); + if (typeof scope.path !== 'string' || !scope.path.startsWith('/')) { + throw new Error('Invalid analytics report'); } - return { - period: JSON.parse(periodData) as VisitReportPeriod, - report, - selection - }; + return query.where('path', '=', scope.path); } diff --git a/packages/db/src/visit-reports/types.test.ts b/packages/db/src/visit-reports/types.test.ts deleted file mode 100644 index eaa3634..0000000 --- a/packages/db/src/visit-reports/types.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { expect, test } from 'vitest'; -import { visitReportKey, zeroVisitCounts } from './types.js'; - -test('zero counts are fresh objects', () => { - const first = zeroVisitCounts(); - const second = zeroVisitCounts(); - first.total = 1; - - expect(second).toEqual({ total: 0, valid: 0, invalid: 0, base: 0 }); - expect(first).not.toBe(second); -}); - -test('scope keys use overview or JSON-encoded domains and exact paths', () => { - expect(visitReportKey()).toBe('overview'); - expect(visitReportKey('go.example.test')).toBe('["go.example.test"]'); - expect(visitReportKey('go.example.test', '/')).toBe('["go.example.test","/"]'); - expect(visitReportKey('go.example.test', '/café,"docs')).toBe( - JSON.stringify(['go.example.test', '/café,"docs']) - ); -}); diff --git a/packages/db/src/visit-reports/types.ts b/packages/db/src/visit-reports/types.ts index 68c9abe..6abeacc 100644 --- a/packages/db/src/visit-reports/types.ts +++ b/packages/db/src/visit-reports/types.ts @@ -1,73 +1,20 @@ -export interface VisitCounts { - total: number; - valid: number; - invalid: number; - base: number; -} - -export function zeroVisitCounts(): VisitCounts { - return { total: 0, valid: 0, invalid: 0, base: 0 }; -} - -export interface TimeBucket { - start: number; - end: number; - partial: boolean; -} - -export interface VisitReportPeriod { - timezone: 'UTC'; - retentionMonths: 3 | 9 | 12; - since: number; - asOf: number; - days: TimeBucket[]; - weeks: TimeBucket[]; - months: TimeBucket[]; -} - -export interface FamilyVisitCounts { - name: string; - counts: VisitCounts; -} - -export interface PreparedVisitReport { - totals: VisitCounts; - daily: VisitCounts[]; - weekly: VisitCounts[]; - monthly: VisitCounts[]; - browsers: FamilyVisitCounts[]; - operatingSystems: FamilyVisitCounts[]; - domainCounts: Array<{ domain: string; counts: VisitCounts }>; - linkCounts: Array<{ path: string; counts: VisitCounts }>; -} +import type { VisitTotals } from '@goto/analytics'; export type ReportInventory = Array<{ domain: string; paths: string[] }>; -export function visitReportKey(domain?: string, path?: string): string { - if (domain === undefined) return 'overview'; - return JSON.stringify(path === undefined ? [domain] : [domain, path]); -} +type DiscriminatedAggregateScope = + | { scopeKind: 'overview'; domain: null; path: null } + | { scopeKind: 'domain'; domain: string; path: null } + | { scopeKind: 'link'; domain: string; path: string }; -export interface SerializedVisitReport { - key: string; - data: string; -} +export type DailyVisitCounts = DiscriminatedAggregateScope & + VisitTotals & { + day: string; + baseValidVisits: number; + baseInvalidVisits: number; + }; -export interface StoredVisitReports { - period: VisitReportPeriod; - report: PreparedVisitReport; - selection: PreparedVisitReport; -} - -export interface DailyVisitCounts extends VisitCounts { - scope: string; - day: string; -} - -export interface ScopedFamilyCounts extends VisitCounts { - scope: string; - name: string; -} +export type ScopedFamilyCounts = DiscriminatedAggregateScope & VisitTotals & { family: string }; export interface AggregatedVisitReports { daily: DailyVisitCounts[]; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce76dfd..a41bdff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -166,6 +166,9 @@ importers: apps/web: dependencies: + '@goto/analytics': + specifier: workspace:* + version: link:../../packages/analytics '@goto/config': specifier: workspace:* version: link:../../packages/config @@ -215,6 +218,9 @@ importers: apps/worker: dependencies: + '@goto/analytics': + specifier: workspace:* + version: link:../../packages/analytics '@goto/config': specifier: workspace:* version: link:../../packages/config @@ -293,6 +299,9 @@ importers: packages/db: dependencies: + '@goto/analytics': + specifier: workspace:* + version: link:../analytics kysely: specifier: 'catalog:' version: 0.28.17 -- 2.54.0