refactor(analytics): simplify JSON report storage #15

Merged
eleith merged 2 commits from feat/analytics-json into main 2026-11-07 19:36:33 +00:00
61 changed files with 2197 additions and 827 deletions

No files matched your search

+7
View File
@@ -0,0 +1,7 @@
---
'@goto/web': patch
'@goto/worker': patch
'@goto/cli': patch
---
Replace opaque analytics cache entries with versioned summary and daily-detail JSON documents identified by report scope. Separate shared analytics contracts and encoding from SQLite persistence. Keep admin reads summary-only, remove stored weekly/monthly series and shared empty/period fallbacks, and rebuild derived reports from retained visits after migration. Existing analytics collection, retention and admin behavior are unchanged.
+1 -1
View File
@@ -7,7 +7,7 @@
"access": "restricted",
"baseBranch": "main",
"updateInternalDependencies": "patch",
"ignore": ["@goto/config", "@goto/db"],
"ignore": ["@goto/analytics", "@goto/config", "@goto/db"],
"privatePackages": {
"version": true,
"tag": true
+7
View File
@@ -9,6 +9,7 @@ on:
- .github/workflows/**
- .woodpecker/**
- packages/db/**
- packages/analytics/**
- packages/config/**
- scripts/**
- tools/**
@@ -20,6 +21,7 @@ on:
- .github/workflows/**
- .woodpecker/**
- packages/db/**
- packages/analytics/**
- packages/config/**
- scripts/**
- tools/**
@@ -48,6 +50,11 @@ jobs:
run: pnpm install --frozen-lockfile
- name: Build DB
run: pnpm --filter '@goto/db...' build
- name: Check analytics dependency
run: |
pnpm --filter @goto/analytics check
pnpm --filter @goto/analytics lint
pnpm --filter @goto/analytics test:coverage
- name: Check types
run: pnpm --filter @goto/db check
- name: Lint DB
+4
View File
@@ -55,6 +55,10 @@ jobs:
run: pnpm --filter @goto/web check
- name: Lint repository
run: pnpm lint
- name: Check analytics dependency
run: |
pnpm --filter @goto/analytics check
pnpm --filter @goto/analytics test:coverage
- name: Unit tests
run: pnpm --filter @goto/web test:coverage
+7
View File
@@ -12,6 +12,7 @@ on:
- apps/worker/**
- packages/config/**
- packages/db/**
- packages/analytics/**
- tools/**
pull_request:
branches: [main]
@@ -23,6 +24,7 @@ on:
- apps/worker/**
- packages/config/**
- packages/db/**
- packages/analytics/**
- tools/**
permissions:
@@ -55,6 +57,11 @@ jobs:
run: pnpm --filter @goto/worker check
- name: Lint worker
run: pnpm --filter @goto/worker lint
- name: Check analytics dependency
run: |
pnpm --filter @goto/analytics check
pnpm --filter @goto/analytics lint
pnpm --filter @goto/analytics test:coverage
- name: Check database dependency
run: |
pnpm --filter @goto/db check
+5
View File
@@ -12,6 +12,7 @@ when:
- .github/workflows/**
- .woodpecker/**
- packages/db/**
- packages/analytics/**
- packages/config/**
- scripts/**
- tools/**
@@ -24,6 +25,7 @@ when:
- .github/workflows/**
- .woodpecker/**
- packages/db/**
- packages/analytics/**
- packages/config/**
- scripts/**
- tools/**
@@ -56,6 +58,8 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics check
- pnpm --filter @goto/analytics lint
- pnpm --filter @goto/db check
- pnpm --filter @goto/db lint
- pnpm exec prettier --check scripts
@@ -68,5 +72,6 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics test:coverage
- pnpm --filter @goto/db test:coverage
- pnpm test:scripts
+2
View File
@@ -50,6 +50,7 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics check
- pnpm --filter @goto/web check
- pnpm lint
@@ -60,6 +61,7 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics test:coverage
- pnpm --filter @goto/web test:coverage
e2e:
+5
View File
@@ -14,6 +14,7 @@ when:
- apps/worker/**
- packages/config/**
- packages/db/**
- packages/analytics/**
- tools/**
- event: pull_request
path:
@@ -25,6 +26,7 @@ when:
- apps/worker/**
- packages/config/**
- packages/db/**
- packages/analytics/**
- tools/**
- event: tag
ref: 'refs/tags/@goto/worker@*'
@@ -50,6 +52,8 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics check
- pnpm --filter @goto/analytics lint
- pnpm --filter @goto/db check
- pnpm --filter @goto/db lint
- pnpm --filter @goto/worker check
@@ -62,6 +66,7 @@ steps:
commands:
- npm install --global corepack@0.36.0
- corepack enable
- pnpm --filter @goto/analytics test:coverage
- pnpm --filter @goto/db test:coverage
- pnpm --filter @goto/worker test:coverage
- pnpm --filter @goto/worker test:smoke
+3 -3
View File
@@ -21,16 +21,16 @@ RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm fetch --frozen-
COPY . .
RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm install --offline --frozen-lockfile
# Only generated output and caches are writable; source binds stay read-only.
RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/db/dist /m/apps/web/.svelte-kit \
RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/analytics/dist /m/packages/db/dist /m/apps/web/.svelte-kit \
'/m/apps/web/node_modules/$app' \
/m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp \
&& chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/db/dist \
&& chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/analytics/dist /m/packages/db/dist \
/m/apps/web/.svelte-kit '/m/apps/web/node_modules/$app' \
/m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp
USER ${LOCAL_UID}:${LOCAL_GID}
WORKDIR /m/apps/web
EXPOSE 5173
CMD ["sh", "-c", "pnpm -r --filter @goto/config --filter @goto/db build && exec node_modules/.bin/vite --host 0.0.0.0 --port 5173 --strictPort"]
CMD ["sh", "-c", "pnpm -r --filter @goto/config --filter @goto/db... build && exec node_modules/.bin/vite --host 0.0.0.0 --port 5173 --strictPort"]
FROM base AS builder
+2
View File
@@ -28,6 +28,7 @@ services:
- ./tsconfig.json:/m/apps/web/tsconfig.json:ro
- ../../packages/config/src:/m/packages/config/src:ro
- ../../packages/db/src:/m/packages/db/src:ro
- ../../packages/analytics/src:/m/packages/analytics/src:ro
restart: unless-stopped
healthcheck:
test:
@@ -64,6 +65,7 @@ services:
- ../worker/src:/m/apps/worker/src:ro
- ../../packages/config/src:/m/packages/config/src:ro
- ../../packages/db/src:/m/packages/db/src:ro
- ../../packages/analytics/src:/m/packages/analytics/src:ro
restart: unless-stopped
healthcheck:
test:
+1
View File
@@ -24,6 +24,7 @@
"test:e2e": "pnpm --filter @goto/worker... build && playwright test"
},
"dependencies": {
"@goto/analytics": "workspace:*",
"@goto/config": "workspace:*",
"@goto/db": "workspace:*",
"better-auth": "catalog:",
+37 -30
View File
@@ -1,22 +1,18 @@
import { access, mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
createVisitPeriod,
encodeVisitReports,
visitDays,
type OverviewSummary,
type VisitDetails
} from '@goto/analytics';
import type { GotoConfiguration } from '@goto/config';
import { openDb, runMigrations } from '@goto/db';
import { publishVisitReports, type PreparedVisitReport } from '@goto/db/visit-reports';
import { publishVisitReports } from '@goto/db/visit-reports';
import { expect, test, vi } from 'vitest';
import { readVisitReport } from './visit-reports.js';
const report: PreparedVisitReport = {
totals: { total: 3, valid: 2, invalid: 1, base: 1 },
daily: [],
weekly: [],
monthly: [],
browsers: [],
operatingSystems: [],
domainCounts: [],
linkCounts: []
};
import { readVisitSummary } from './visit-reports.js';
test('reads prepared output and hides snapshots from a different retention policy', async () => {
const directory = await mkdtemp(join(tmpdir(), 'goto-page-reports-'));
@@ -27,29 +23,40 @@ test('reads prepared output and hides snapshots from a different retention polic
},
analytics: { retention_months: 3 }
};
const period = createVisitPeriod(3, Date.parse('2026-06-15T12:00:00.000Z'));
const totals = { validVisits: 2, invalidVisits: 1 };
const summary: OverviewSummary = {
version: 1,
scope: { kind: 'overview' },
period,
totals,
base: { validVisits: 1, invalidVisits: 0 },
domains: [{ domain: 'go.example.test', totals }]
};
const days = visitDays(period);
const details: VisitDetails = {
version: 1,
scope: summary.scope,
period,
daily: days.map((day, index) => ({
...day,
validVisits: index === days.length - 1 ? totals.validVisits : 0,
invalidVisits: index === days.length - 1 ? totals.invalidVisits : 0
})),
browsers: [{ family: 'Other', ...totals }],
operatingSystems: [{ family: 'Other', ...totals }]
};
try {
const db = await openDb(configuration.database.path);
try {
await runMigrations(db);
const period = {
timezone: 'UTC' as const,
retentionMonths: 3 as const,
since: 100,
asOf: 200,
days: [],
weeks: [],
months: []
};
await publishVisitReports(db, period, [
{ key: 'overview', data: JSON.stringify(report) },
{ key: 'empty', data: JSON.stringify(report) }
]);
await publishVisitReports(db, encodeVisitReports([{ summary, details }]));
} finally {
await db.destroy();
}
expect((await readVisitReport(configuration))?.report.totals.total).toBe(3);
expect(await readVisitSummary(configuration, { kind: 'overview' })).toEqual(summary);
configuration.analytics.retention_months = 9;
expect(await readVisitReport(configuration)).toBeNull();
expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull();
} finally {
await rm(directory, { recursive: true, force: true });
}
@@ -67,7 +74,7 @@ test('enabled collection fails safely without creating missing storage', async (
},
analytics: { retention_months: 3 as const }
};
expect(await readVisitReport(configuration)).toBeNull();
expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull();
expect(warning).toHaveBeenCalledExactlyOnceWith('Prepared visit reports unavailable');
await expect(access(missing)).rejects.toMatchObject({ code: 'ENOENT' });
} finally {
@@ -83,7 +90,7 @@ test('disabled collection does not attempt to open missing storage', async () =>
database: { path: '/missing/not-used.sqlite', worker_path: '/missing/worker.sqlite' },
analytics: { retention_months: 0 as const }
};
expect(await readVisitReport(configuration)).toBeNull();
expect(await readVisitSummary(configuration, { kind: 'overview' })).toBeNull();
expect(warning).not.toHaveBeenCalled();
} finally {
warning.mockRestore();
+8 -8
View File
@@ -1,23 +1,23 @@
import type { ReportScope, VisitSummary } from '@goto/analytics';
import type { GotoConfiguration } from '@goto/config';
import { openDb } from '@goto/db';
import { readPreparedVisitReports, type StoredVisitReports } from '@goto/db/visit-reports';
import { readVisitSummary as readPreparedSummary } from '@goto/db/visit-reports';
import type { Immutable } from './snapshot.js';
type ReportConfiguration = Immutable<Pick<GotoConfiguration, 'database' | 'analytics'>>;
export async function readVisitReport(
export async function readVisitSummary(
configuration: ReportConfiguration,
domain?: string,
path?: string
): Promise<StoredVisitReports | null> {
scope: ReportScope
): Promise<VisitSummary | null> {
const months = configuration.analytics.retention_months;
if (months === 0) return null;
let db: Awaited<ReturnType<typeof openDb>> | undefined;
try {
db = await openDb(configuration.database.path, { readOnly: true });
const reports = await readPreparedVisitReports(db, domain, path);
if (!reports || reports.period.retentionMonths !== months) return null;
return reports;
const summary = await readPreparedSummary(db, scope);
if (!summary || summary.period.retentionMonths !== months) return null;
return summary;
} catch {
warnUnavailable();
return null;
@@ -1,5 +1,5 @@
import { readVisitReport } from '#lib/server/visit-reports.js';
import type { StoredVisitReports } from '@goto/db/visit-reports';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import type { OverviewSummary } from '@goto/analytics';
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = async ({ locals }) => {
@@ -7,25 +7,27 @@ export const load: PageServerLoad = async ({ locals }) => {
const configuration = snapshot.configuration;
const retentionMonths = configuration.analytics.retention_months;
const collectionEnabled = retentionMonths !== 0;
let preparedReports: StoredVisitReports | null = null;
if (collectionEnabled) preparedReports = await readVisitReport(configuration);
const report = preparedReports?.report;
let overview: OverviewSummary | null = null;
if (collectionEnabled) {
const summary = await readVisitSummary(configuration, { kind: 'overview' });
if (summary && 'domains' in summary) overview = summary;
}
const countsByDomain = new Map(overview?.domains.map((row) => [row.domain, row.totals]));
const domains = Object.entries(snapshot.domains).map(([domain, prepared]) => ({
domain,
linkCount: Object.keys(prepared.links).length,
counts: report?.domainCounts.find((row) => row.domain === domain)?.counts ?? null,
counts: countsByDomain.get(domain) ?? null,
href: `/admin/domains/${encodeURIComponent(domain)}`
}));
const asOf = overview ? Date.parse(overview.period.asOf) : null;
return {
collectionEnabled,
available: preparedReports !== null,
available: overview !== null,
retentionMonths,
since: preparedReports?.period.since ?? null,
asOf: preparedReports?.period.asOf ?? null,
asOf,
domainCount: domains.length,
linkCount: domains.reduce((sum, row) => sum + row.linkCount, 0),
totals: report?.totals ?? null,
totals: overview?.totals ?? null,
domains
};
};
@@ -12,7 +12,7 @@
<div class="admin-cards admin-cards-summary">
<StatCard label="Domains" value={data.domainCount} />
<StatCard label="Links" value={data.linkCount} />
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
<StatCard label="Valid visits" value={data.totals?.validVisits ?? null} />
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
</div>
<section>
@@ -31,7 +31,7 @@
<tr>
<th scope="row"><a class="admin-row-link" href={row.href}>{row.domain}</a></th>
<td>{formatCount(row.linkCount)}</td>
<td>{formatCount(row.counts?.valid ?? null)}</td>
<td>{formatCount(row.counts?.validVisits ?? null)}</td>
</tr>
{/each}
</tbody>
@@ -1,7 +1,7 @@
import { error } from '@sveltejs/kit';
import type { VisitCounts, StoredVisitReports } from '@goto/db/visit-reports';
import type { DomainSummary, VisitTotals } from '@goto/analytics';
import { currentLinkState } from '#lib/server/link-state.js';
import { readVisitReport } from '#lib/server/visit-reports.js';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = async ({ locals, params }) => {
@@ -11,15 +11,14 @@ export const load: PageServerLoad = async ({ locals, params }) => {
const prepared = snapshot.domains[domain]!;
const configuration = snapshot.configuration;
const collectionEnabled = configuration.analytics.retention_months !== 0;
let preparedReports: StoredVisitReports | null = null;
if (collectionEnabled) preparedReports = await readVisitReport(configuration, domain);
const report = preparedReports?.report;
const countsByPath = new Map<string, VisitCounts>();
for (const row of report?.linkCounts ?? []) countsByPath.set(row.path, row.counts);
const baseCounts = countsByPath.get('/') ?? null;
let invalidClicks: number | null = null;
if (report && baseCounts) invalidClicks = report.totals.invalid - baseCounts.invalid;
let domainSummary: DomainSummary | null = null;
if (collectionEnabled) {
const scope = { kind: 'domain' as const, domain };
const summary = await readVisitSummary(configuration, scope);
if (summary && 'links' in summary) domainSummary = summary;
}
const countsByPath = new Map<string, VisitTotals>();
for (const row of domainSummary?.links ?? []) countsByPath.set(row.path, row.totals);
const now = Date.now();
const links = Object.entries(prepared.links).map(([path, link]) => ({
path,
@@ -35,21 +34,25 @@ export const load: PageServerLoad = async ({ locals, params }) => {
baseUrl = root.url;
baseState = 'Active';
}
let invalidClicks: number | null = null;
if (domainSummary) {
invalidClicks = domainSummary.totals.invalidVisits - domainSummary.base.invalidVisits;
}
const asOf = domainSummary ? Date.parse(domainSummary.period.asOf) : null;
return {
domain,
retentionMonths: configuration.analytics.retention_months,
collectionEnabled,
available: preparedReports !== null,
since: preparedReports?.period.since ?? null,
asOf: preparedReports?.period.asOf ?? null,
available: domainSummary !== null,
asOf,
linkCount: links.length,
totals: report?.totals ?? null,
totals: domainSummary?.totals ?? null,
invalidClicks,
base: {
url: baseUrl,
state: baseState,
counts: baseCounts,
counts: domainSummary?.base ?? null,
href: linkHref(domain, '/')
},
links
@@ -11,7 +11,7 @@
<ReportNotice collectionEnabled={data.collectionEnabled} available={data.available} />
<div class="admin-cards admin-cards-summary">
<StatCard label="Links" value={data.linkCount} />
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
<StatCard label="Valid visits" value={data.totals?.validVisits ?? null} />
<StatCard label="Invalid clicks" value={data.invalidClicks} />
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
</div>
@@ -35,7 +35,7 @@
>{:else}404 Not found{/if}</td
>
<td>{data.base.state}</td>
<td>{formatCount(data.base.counts?.valid ?? null)}</td>
<td>{formatCount(data.base.counts?.validVisits ?? null)}</td>
</tr>
</tbody>
</table>
@@ -60,7 +60,7 @@
><a class="admin-row-link" href={link.href}><code>{link.path}</code></a></th
>
<td><a class="admin-destination" href={link.url}>{link.url}</a></td>
<td>{link.state}</td><td>{formatCount(link.counts?.valid ?? null)}</td>
<td>{link.state}</td><td>{formatCount(link.counts?.validVisits ?? null)}</td>
</tr>
{/each}
</tbody>
@@ -1,7 +1,7 @@
import { error } from '@sveltejs/kit';
import type { StoredVisitReports } from '@goto/db/visit-reports';
import type { VisitSummary } from '@goto/analytics';
import { currentLinkState } from '#lib/server/link-state.js';
import { readVisitReport } from '#lib/server/visit-reports.js';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import type { PageServerLoad } from './$types';
export const load: PageServerLoad = async ({ locals, params }) => {
@@ -15,9 +15,13 @@ export const load: PageServerLoad = async ({ locals, params }) => {
const link = prepared.links[path];
const configuration = snapshot.configuration;
const collectionEnabled = configuration.analytics.retention_months !== 0;
let reports: StoredVisitReports | null = null;
if (collectionEnabled) reports = await readVisitReport(configuration, domain, path);
let linkSummary: VisitSummary | null = null;
if (collectionEnabled) {
const scope = { kind: 'link' as const, domain, path };
linkSummary = await readVisitSummary(configuration, scope);
}
const state = link ? currentLinkState(link, Date.now()) : 'Fallback';
const asOf = linkSummary ? Date.parse(linkSummary.period.asOf) : null;
return {
domain,
@@ -28,9 +32,8 @@ export const load: PageServerLoad = async ({ locals, params }) => {
state,
retentionMonths: configuration.analytics.retention_months,
collectionEnabled,
available: reports !== null,
since: reports?.period.since ?? null,
asOf: reports?.period.asOf ?? null,
totals: reports?.selection.totals ?? null
available: linkSummary !== null,
asOf,
totals: linkSummary?.totals ?? null
};
};
@@ -16,7 +16,7 @@
>
<ReportNotice collectionEnabled={data.collectionEnabled} available={data.available} />
<div class="admin-cards">
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
<StatCard label="Valid visits" value={data.totals?.validVisits ?? null} />
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
</div>
<section class="admin-info" aria-label="Link details">
@@ -1,11 +1,11 @@
import { validateConfig } from '@goto/config';
import type { PreparedVisitReport } from '@goto/db/visit-reports';
import type { LinkSummary } from '@goto/analytics';
import { beforeEach, expect, test, vi } from 'vitest';
import { prepareSnapshot } from '#lib/server/snapshot.js';
import { readVisitReport } from '#lib/server/visit-reports.js';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import { load } from './+page.server.js';
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() }));
beforeEach(() => vi.resetAllMocks());
const domain = 'go.example.test';
@@ -31,47 +31,41 @@ function request(path = '/docs', host = domain, retention_months = 3) {
params: { domain: host, path: path.slice(1) }
} as Parameters<typeof load>[0];
}
const report: PreparedVisitReport = {
totals: { total: 6, valid: 3, invalid: 3, base: 1 },
daily: [],
weekly: [],
monthly: [],
browsers: [],
operatingSystems: [],
domainCounts: [],
linkCounts: []
};
const period = {
timezone: 'UTC' as const,
retentionMonths: 3 as const,
since: 100,
asOf: 200,
days: [],
weeks: [],
months: []
const summary: LinkSummary = {
version: 1,
scope: { kind: 'link', domain, path: '/docs' },
period: {
from: '2026-03-15T12:00:00.000Z',
asOf: '2026-06-15T12:00:00.000Z',
timezone: 'UTC',
retentionMonths: 3
},
totals: { validVisits: 2, invalidVisits: 1 }
};
test('loads the requested path report and current destination with a domain breadcrumb', async () => {
const totals = { total: 3, valid: 2, invalid: 1, base: 0 };
vi.mocked(readVisitReport).mockResolvedValue({
period,
report,
selection: { ...report, totals }
});
test('loads the requested path summary and current destination with a domain breadcrumb', async () => {
vi.mocked(readVisitSummary).mockResolvedValue(summary);
const data = await load(request());
if (!data) throw new Error('Missing link page');
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/docs');
expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), {
kind: 'link',
domain,
path: '/docs'
});
expect(data.path).toBe('/docs');
expect(data.totals).toEqual(totals);
expect(data.totals).toEqual(summary.totals);
expect(data.url).toBe('https://example.test/docs');
expect(data.state).toBe('Active');
expect(data.domainHref).toBe('/admin/domains/go.example.test');
expect(data.asOf).toBe(200);
expect(data.asOf).toBe(Date.parse(summary.period.asOf));
expect(data.retentionMonths).toBe(3);
});
test('keeps decoded nested and literal-percent paths distinct and supports the unconfigured root', async () => {
vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report });
vi.mocked(readVisitSummary)
.mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/a/b' } })
.mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/a%2Fb' } })
.mockResolvedValueOnce({ ...summary, scope: { kind: 'link', domain, path: '/' } });
const nested = await load(request('/a/b'));
const percent = await load(request('/a%2Fb'));
const root = await load(request('/'));
@@ -79,7 +73,21 @@ test('keeps decoded nested and literal-percent paths distinct and supports the u
expect(nested.path).toBe('/a/b');
expect(percent.path).toBe('/a%2Fb');
expect(percent.state).toBe('Disabled');
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/a%2Fb');
expect(readVisitSummary).toHaveBeenNthCalledWith(1, expect.any(Object), {
kind: 'link',
domain,
path: '/a/b'
});
expect(readVisitSummary).toHaveBeenNthCalledWith(2, expect.any(Object), {
kind: 'link',
domain,
path: '/a%2Fb'
});
expect(readVisitSummary).toHaveBeenNthCalledWith(3, expect.any(Object), {
kind: 'link',
domain,
path: '/'
});
expect(root.path).toBe('/');
expect(root.state).toBe('Fallback');
expect(root.url).toBeNull();
@@ -89,8 +97,8 @@ test('keeps decoded nested and literal-percent paths distinct and supports the u
test('rejects unknown scopes before reading and keeps unavailable or disabled counts empty', async () => {
await expect(load(request('/docs', 'unknown.test'))).rejects.toMatchObject({ status: 404 });
await expect(load(request('/not-configured'))).rejects.toMatchObject({ status: 404 });
expect(readVisitReport).not.toHaveBeenCalled();
vi.mocked(readVisitReport).mockResolvedValue(null);
expect(readVisitSummary).not.toHaveBeenCalled();
vi.mocked(readVisitSummary).mockResolvedValue(null);
const missing = await load(request());
if (!missing) throw new Error('Missing unavailable page');
expect(missing.available).toBe(false);
@@ -101,5 +109,5 @@ test('rejects unknown scopes before reading and keeps unavailable or disabled co
expect(disabled.collectionEnabled).toBe(false);
expect(disabled.retentionMonths).toBe(0);
expect(disabled.totals).toBeNull();
expect(readVisitReport).not.toHaveBeenCalled();
expect(readVisitSummary).not.toHaveBeenCalled();
});
@@ -1,11 +1,11 @@
import { validateConfig } from '@goto/config';
import type { PreparedVisitReport, StoredVisitReports } from '@goto/db/visit-reports';
import type { DomainSummary } from '@goto/analytics';
import { beforeEach, expect, test, vi } from 'vitest';
import { prepareSnapshot } from '#lib/server/snapshot.js';
import { readVisitReport } from '#lib/server/visit-reports.js';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import { load } from './+page.server.js';
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() }));
beforeEach(() => vi.resetAllMocks());
const domain = 'go.example.test';
@@ -28,59 +28,53 @@ function request(host = domain, rootUrl?: string) {
params: { domain: host }
} as Parameters<typeof load>[0];
}
const docs = { total: 3, valid: 2, invalid: 1, base: 0 };
const root = { total: 1, valid: 0, invalid: 1, base: 1 };
const empty = { total: 0, valid: 0, invalid: 0, base: 0 };
const report: PreparedVisitReport = {
totals: { total: 6, valid: 3, invalid: 3, base: 1 },
daily: [],
weekly: [],
monthly: [],
browsers: [],
operatingSystems: [],
domainCounts: [],
linkCounts: [
{ path: '/docs', counts: docs },
{ path: '/', counts: root },
{ path: '/a%2Fb', counts: empty }
const docs = { validVisits: 2, invalidVisits: 1 };
const root = { validVisits: 0, invalidVisits: 1 };
const empty = { validVisits: 0, invalidVisits: 0 };
const summary: DomainSummary = {
version: 1,
scope: { kind: 'domain', domain },
period: {
from: '2026-03-15T12:00:00.000Z',
asOf: '2026-06-15T12:00:00.000Z',
timezone: 'UTC',
retentionMonths: 3
},
totals: { validVisits: 3, invalidVisits: 3 },
base: root,
links: [
{ path: '/docs', totals: docs },
{ path: '/', totals: root },
{ path: '/a%2Fb', totals: empty }
]
};
const period = {
timezone: 'UTC' as const,
retentionMonths: 3 as const,
since: 100,
asOf: 200,
days: [],
weeks: [],
months: []
};
test('reads only domain-wide traffic and links to dedicated path routes', async () => {
const reports: StoredVisitReports = { period, report, selection: report };
vi.mocked(readVisitReport).mockResolvedValue(reports);
vi.mocked(readVisitSummary).mockResolvedValue(summary);
const data = await load(request());
if (!data) throw new Error('Missing page data');
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain);
expect(data.totals).toEqual(report.totals);
expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), {
kind: 'domain',
domain
});
expect(data.totals).toEqual(summary.totals);
expect(data.invalidClicks).toBe(2);
expect(data.linkCount).toBe(3);
expect(data.retentionMonths).toBe(3);
expect(data.links).toMatchObject([
{ href: '/admin/domains/go.example.test/links/docs' },
{ href: '/admin/domains/go.example.test/links/docs', counts: docs },
{ href: '/admin/domains/go.example.test/links/a/b' },
{ href: '/admin/domains/go.example.test/links/a%252Fb' }
{ href: '/admin/domains/go.example.test/links/a%252Fb', counts: empty, state: 'Disabled' }
]);
expect(data.links[2]?.counts).toEqual(empty);
expect(data.links[2]?.state).toBe('Disabled');
expect(data.base.href).toBe('/admin/domains/go.example.test/links');
expect(data.base.url).toBeNull();
expect(data.base.state).toBe('Fallback');
expect(data.base.counts).toEqual(root);
expect(data.asOf).toBe(200);
expect(data.asOf).toBe(Date.parse(summary.period.asOf));
});
test('shows an active base destination without reclassifying historical outcomes', async () => {
vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report });
vi.mocked(readVisitSummary).mockResolvedValue(summary);
const data = await load(request(domain, 'https://example.test/start'));
if (!data) throw new Error('Missing base page data');
expect(data.base.url).toBe('https://example.test/start');
@@ -89,29 +83,24 @@ test('shows an active base destination without reclassifying historical outcomes
});
test('excludes invalid base traffic without subtracting valid base redirects', async () => {
const mixedRoot = { total: 5, valid: 3, invalid: 2, base: 5 };
const mixedReport: PreparedVisitReport = {
...report,
totals: { total: 10, valid: 6, invalid: 4, base: 5 },
linkCounts: report.linkCounts.map((row) =>
row.path === '/' ? { ...row, counts: mixedRoot } : row
)
const mixedRoot = { validVisits: 3, invalidVisits: 2 };
const mixedSummary: DomainSummary = {
...summary,
totals: { validVisits: 6, invalidVisits: 4 },
base: mixedRoot,
links: summary.links.map((row) => (row.path === '/' ? { ...row, totals: mixedRoot } : row))
};
vi.mocked(readVisitReport).mockResolvedValue({
period,
report: mixedReport,
selection: mixedReport
});
vi.mocked(readVisitSummary).mockResolvedValue(mixedSummary);
const data = await load(request());
if (!data) throw new Error('Missing domain page data');
expect(data.invalidClicks).toBe(2);
expect(data.totals?.valid).toBe(6);
expect(data.totals?.validVisits).toBe(6);
});
test('rejects unknown domains before reading and preserves unavailable rather than zero', async () => {
await expect(load(request('unknown.test'))).rejects.toMatchObject({ status: 404 });
expect(readVisitReport).not.toHaveBeenCalled();
vi.mocked(readVisitReport).mockResolvedValue(null);
expect(readVisitSummary).not.toHaveBeenCalled();
vi.mocked(readVisitSummary).mockResolvedValue(null);
const data = await load(request());
if (!data) throw new Error('Missing unavailable page');
expect(data.available).toBe(false);
@@ -1,11 +1,11 @@
import { validateConfig } from '@goto/config';
import type { StoredVisitReports } from '@goto/db/visit-reports';
import type { OverviewSummary } from '@goto/analytics';
import { beforeEach, expect, test, vi } from 'vitest';
import { prepareSnapshot } from '#lib/server/snapshot.js';
import { readVisitReport } from '#lib/server/visit-reports.js';
import { readVisitSummary } from '#lib/server/visit-reports.js';
import { load } from './+page.server.js';
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitSummary: vi.fn() }));
beforeEach(() => vi.resetAllMocks());
function request(retention_months = 3) {
@@ -27,39 +27,35 @@ function request(retention_months = 3) {
return { locals: { snapshot: prepareSnapshot(prepared) } } as Parameters<typeof load>[0];
}
test('shows current configuration and the worker-prepared snapshot without re-aggregating', async () => {
const counts = { total: 4, valid: 2, invalid: 2, base: 1 };
const report = {
totals: { ...counts, total: 9, valid: 7 },
daily: [],
weekly: [],
monthly: [],
browsers: [],
operatingSystems: [],
domainCounts: [{ domain: 'go.example.test', counts }],
linkCounts: []
};
const prepared: StoredVisitReports = {
test('shows current configuration and the worker-prepared summary without re-aggregating', async () => {
const counts = { validVisits: 2, invalidVisits: 2 };
const summary: OverviewSummary = {
version: 1,
scope: { kind: 'overview' },
period: {
from: '2026-03-15T12:00:00.000Z',
asOf: '2026-06-15T12:00:00.000Z',
timezone: 'UTC',
retentionMonths: 3,
since: 100,
asOf: 200,
days: [],
weeks: [],
months: []
retentionMonths: 3
},
report,
selection: report
totals: { validVisits: 7, invalidVisits: 2 },
base: { validVisits: 0, invalidVisits: 1 },
domains: [{ domain: 'go.example.test', totals: counts }]
};
vi.mocked(readVisitReport).mockResolvedValue(prepared);
vi.mocked(readVisitSummary).mockResolvedValue(summary);
const data = await load(request());
if (!data) throw new Error('Missing page data');
expect(readVisitSummary).toHaveBeenCalledExactlyOnceWith(expect.any(Object), {
kind: 'overview'
});
expect(data.domainCount).toBe(1);
expect(data.linkCount).toBe(2);
expect(data.totals?.total).toBe(9);
expect(data.asOf).toBe(200);
expect(data.domains[0]?.href).toBe('/admin/domains/go.example.test');
expect(data.totals).toEqual(summary.totals);
expect(data.asOf).toBe(Date.parse(summary.period.asOf));
expect(data.domains[0]).toMatchObject({
href: '/admin/domains/go.example.test',
counts
});
});
test('disabled collection does not read a report; missing preparation is not zero', async () => {
@@ -67,8 +63,8 @@ test('disabled collection does not read a report; missing preparation is not zer
if (!disabled) throw new Error('Missing disabled page');
expect(disabled.collectionEnabled).toBe(false);
expect(disabled.totals).toBeNull();
expect(readVisitReport).not.toHaveBeenCalled();
vi.mocked(readVisitReport).mockResolvedValue(null);
expect(readVisitSummary).not.toHaveBeenCalled();
vi.mocked(readVisitSummary).mockResolvedValue(null);
const unavailable = await load(request());
if (!unavailable) throw new Error('Missing unavailable page');
expect(unavailable.available).toBe(false);
+1
View File
@@ -18,6 +18,7 @@
"test:smoke": "node e2e/startup.js"
},
"dependencies": {
"@goto/analytics": "workspace:*",
"@goto/config": "workspace:*",
"@goto/db": "workspace:*",
"openworkflow": "catalog:",
@@ -1,47 +1,233 @@
import type { PreparedVisitReport } from '@goto/db/visit-reports';
import { createVisitPeriod, visitDays } from '@goto/analytics';
import { expect, test } from 'vitest';
import { buildVisitReports } from './build-visit-reports.js';
import { reportPeriod } from './report-buckets.js';
test('places counts on the correct UTC days, Monday weeks and calendar months', () => {
const period = reportPeriod(3, Date.parse('2026-07-01T12:00:00Z'));
const reports = buildVisitReports(
test('builds gap-free UTC daily details with zeros and partial calendar edges', () => {
const period = createVisitPeriod(3, Date.parse('2026-05-31T12:34:56.789Z'));
const [report] = buildVisitReports(
{
daily: [
{ scope: 'overview', day: '2026-05-31', total: 3, valid: 2, invalid: 1, base: 1 },
{ scope: 'overview', day: '2026-06-01', total: 5, valid: 1, invalid: 4, base: 2 }
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-02-28',
validVisits: 2,
invalidVisits: 1,
baseValidVisits: 1,
baseInvalidVisits: 0
},
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-05-31',
validVisits: 1,
invalidVisits: 4,
baseValidVisits: 0,
baseInvalidVisits: 2
}
],
browsers: [],
operatingSystems: []
browsers: [
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Firefox',
validVisits: 3,
invalidVisits: 5
}
],
operatingSystems: [
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Linux',
validVisits: 3,
invalidVisits: 5
}
]
},
[],
period
);
const report = JSON.parse(
reports.find((entry) => entry.key === 'overview')!.data
) as PreparedVisitReport;
const firstDay = period.days.findIndex(
(bucket) => bucket.start === Date.parse('2026-05-30T00:00:00Z')
);
const firstWeek = period.weeks.findIndex(
(bucket) => bucket.start === Date.parse('2026-05-25T00:00:00Z')
);
expect(report.daily.slice(firstDay, firstDay + 4)).toEqual([
{ total: 0, valid: 0, invalid: 0, base: 0 },
{ total: 3, valid: 2, invalid: 1, base: 1 },
{ total: 5, valid: 1, invalid: 4, base: 2 },
{ total: 0, valid: 0, invalid: 0, base: 0 }
expect(report?.summary).toEqual({
version: 1,
scope: { kind: 'overview' },
period,
totals: { validVisits: 3, invalidVisits: 5 },
base: { validVisits: 1, invalidVisits: 2 },
domains: []
});
expect(report?.details.daily).toHaveLength(visitDays(period).length);
expect(report?.details.daily[0]).toEqual({
start: '2026-02-28T00:00:00.000Z',
end: '2026-03-01T00:00:00.000Z',
partial: true,
validVisits: 2,
invalidVisits: 1
});
expect(report?.details.daily[1]).toEqual({
start: '2026-03-01T00:00:00.000Z',
end: '2026-03-02T00:00:00.000Z',
partial: false,
validVisits: 0,
invalidVisits: 0
});
expect(report?.details.daily.at(-1)).toEqual({
start: '2026-05-31T00:00:00.000Z',
end: '2026-06-01T00:00:00.000Z',
partial: true,
validVisits: 1,
invalidVisits: 4
});
});
test('indexes explicit domain and link scopes, including empty links and root, without mixing families', () => {
const period = createVisitPeriod(3, Date.parse('2026-07-01T12:00:00Z'));
const overview = { scopeKind: 'overview' as const, domain: null, path: null };
const domain = { scopeKind: 'domain' as const, domain: 'go.example.test', path: null };
const link = { scopeKind: 'link' as const, domain: 'go.example.test', path: '/docs' };
const root = { ...link, path: '/' };
const reports = buildVisitReports(
{
daily: [
{
...overview,
day: '2026-06-01',
validVisits: 8,
invalidVisits: 3,
baseValidVisits: 1,
baseInvalidVisits: 1
},
{
...domain,
day: '2026-06-01',
validVisits: 4,
invalidVisits: 2,
baseValidVisits: 1,
baseInvalidVisits: 1
},
{
...link,
day: '2026-06-01',
validVisits: 2,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
...root,
day: '2026-06-01',
validVisits: 1,
invalidVisits: 1,
baseValidVisits: 1,
baseInvalidVisits: 1
}
],
browsers: [
{ ...overview, family: 'Other', validVisits: 8, invalidVisits: 3 },
{ ...domain, family: 'Firefox', validVisits: 4, invalidVisits: 2 },
{ ...link, family: 'Chrome', validVisits: 2, invalidVisits: 0 },
{ ...root, family: 'Firefox', validVisits: 1, invalidVisits: 1 }
],
operatingSystems: [
{ ...overview, family: 'Other', validVisits: 8, invalidVisits: 3 },
{ ...domain, family: 'Linux', validVisits: 4, invalidVisits: 2 },
{ ...link, family: 'Linux', validVisits: 2, invalidVisits: 0 },
{ ...root, family: 'Linux', validVisits: 1, invalidVisits: 1 }
]
},
[
{ domain: 'go.example.test', paths: ['/docs', '/empty', '/'] },
{ domain: 'empty.example.test', paths: ['/docs'] }
],
period
);
const docs = reports.find(
({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/docs'
);
const base = reports.find(
({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/'
);
const empty = reports.find(
({ summary }) => summary.scope.kind === 'link' && summary.scope.path === '/empty'
);
const domainReport = reports.find(({ summary }) => summary.scope.kind === 'domain');
const overviewReport = reports.find(({ summary }) => summary.scope.kind === 'overview');
expect(reports).toHaveLength(8);
expect(domainReport?.summary).toEqual({
version: 1,
scope: { kind: 'domain', domain: 'go.example.test' },
period,
totals: { validVisits: 4, invalidVisits: 2 },
base: { validVisits: 1, invalidVisits: 1 },
links: [
{ path: '/docs', totals: { validVisits: 2, invalidVisits: 0 } },
{ path: '/empty', totals: { validVisits: 0, invalidVisits: 0 } },
{ path: '/', totals: { validVisits: 1, invalidVisits: 1 } }
]
});
expect(overviewReport?.summary).toEqual({
version: 1,
scope: { kind: 'overview' },
period,
totals: { validVisits: 8, invalidVisits: 3 },
base: { validVisits: 1, invalidVisits: 1 },
domains: [
{ domain: 'go.example.test', totals: { validVisits: 4, invalidVisits: 2 } },
{ domain: 'empty.example.test', totals: { validVisits: 0, invalidVisits: 0 } }
]
});
expect(docs?.summary).toEqual({
version: 1,
scope: { kind: 'link', domain: 'go.example.test', path: '/docs' },
period,
totals: { validVisits: 2, invalidVisits: 0 }
});
expect(docs?.details.browsers).toEqual([{ family: 'Chrome', validVisits: 2, invalidVisits: 0 }]);
expect(domainReport?.details.browsers).toEqual([
{ family: 'Firefox', validVisits: 4, invalidVisits: 2 }
]);
expect(report.weekly.slice(firstWeek, firstWeek + 2)).toEqual([
{ total: 3, valid: 2, invalid: 1, base: 1 },
{ total: 5, valid: 1, invalid: 4, base: 2 }
expect(base?.details.operatingSystems).toEqual([
{ family: 'Linux', validVisits: 1, invalidVisits: 1 }
]);
expect(report.monthly).toEqual([
{ total: 0, valid: 0, invalid: 0, base: 0 },
{ total: 3, valid: 2, invalid: 1, base: 1 },
{ total: 5, valid: 1, invalid: 4, base: 2 },
{ total: 0, valid: 0, invalid: 0, base: 0 }
]);
expect(report.totals).toEqual({ total: 8, valid: 3, invalid: 5, base: 3 });
expect(empty?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 });
expect(empty?.details.daily).toEqual(
visitDays(period).map((day) => ({ ...day, validVisits: 0, invalidVisits: 0 }))
);
expect(empty?.details.browsers).toEqual([]);
const otherDomainDocs = reports.find(
({ summary }) =>
summary.scope.kind === 'link' &&
summary.scope.domain === 'empty.example.test' &&
summary.scope.path === '/docs'
);
expect(otherDomainDocs?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 });
const otherDomain = reports.find(
({ summary }) =>
summary.scope.kind === 'domain' && summary.scope.domain === 'empty.example.test'
);
expect(otherDomain?.summary).toEqual({
version: 1,
scope: { kind: 'domain', domain: 'empty.example.test' },
period,
totals: { validVisits: 0, invalidVisits: 0 },
base: { validVisits: 0, invalidVisits: 0 },
links: [{ path: '/docs', totals: { validVisits: 0, invalidVisits: 0 } }]
});
const unconfiguredRoot = reports.find(
({ summary }) =>
summary.scope.kind === 'link' &&
summary.scope.domain === 'empty.example.test' &&
summary.scope.path === '/'
);
expect(unconfiguredRoot?.summary.totals).toEqual({ validVisits: 0, invalidVisits: 0 });
for (const report of reports) {
expect(report.details.scope).toEqual(report.summary.scope);
expect(report.details.period).toEqual(period);
}
});
+155 -102
View File
@@ -1,14 +1,20 @@
import {
visitReportKey,
zeroVisitCounts,
type AggregatedVisitReports,
type DailyVisitCounts,
type ScopedFamilyCounts,
visitDays,
zeroVisitTotals,
type DomainSummary,
type LinkSummary,
type OverviewSummary,
type PreparedVisitReport,
type ReportInventory,
type SerializedVisitReport,
type VisitCounts,
type VisitReportPeriod
type ReportPeriod,
type ReportScope,
type VisitDetails,
type VisitTotals
} from '@goto/analytics';
import type {
AggregatedVisitReports,
DailyVisitCounts,
ReportInventory,
ScopedFamilyCounts
} from '@goto/db/visit-reports';
interface ScopeCounts {
@@ -17,117 +23,164 @@ interface ScopeCounts {
operatingSystems: ScopedFamilyCounts[];
}
interface DomainCounts {
counts: ScopeCounts;
links: Map<string, ScopeCounts>;
}
interface ReportCounts {
overview: ScopeCounts;
domains: Map<string, DomainCounts>;
}
export function buildVisitReports(
aggregates: AggregatedVisitReports,
inventory: ReportInventory,
period: VisitReportPeriod
): SerializedVisitReport[] {
period: ReportPeriod
): PreparedVisitReport[] {
const indexed = indexCounts(aggregates);
const empty = emptyReport(period);
const reports: SerializedVisitReport[] = [{ key: 'empty', data: JSON.stringify(empty) }];
const linkTotals = new Map<string, VisitCounts>();
const domainTotals = new Map<string, VisitCounts>();
const days = visitDays(period);
const reports: PreparedVisitReport[] = [];
const domains: OverviewSummary['domains'] = [];
for (const entry of inventory) {
const paths = [...new Set([...entry.paths, '/'])];
for (const path of paths) {
const key = visitReportKey(entry.domain, path);
const source = indexed.get(key);
if (!source) {
linkTotals.set(key, zeroVisitCounts());
continue;
}
const report = scopeReport(source, period);
linkTotals.set(key, report.totals);
reports.push({ key, data: JSON.stringify(report) });
const domain = indexed.domains.get(entry.domain);
const links: DomainSummary['links'] = [];
const configuredPaths = new Set(entry.paths);
const allPaths = new Set([...configuredPaths, '/']);
for (const path of allPaths) {
const counts = domain?.links.get(path) ?? emptyCounts();
const scope = { kind: 'link' as const, domain: entry.domain, path };
const summary = buildLinkSummary(scope, counts, period);
if (configuredPaths.has(path)) links.push({ path, totals: summary.totals });
reports.push({ summary, details: buildDetails(scope, counts, period, days) });
}
const key = visitReportKey(entry.domain);
const source = indexed.get(key);
const report = source ? scopeReport(source, period) : { ...empty };
report.linkCounts = paths.map((path) => {
const linkKey = visitReportKey(entry.domain, path);
return { path, counts: linkTotals.get(linkKey) ?? zeroVisitCounts() };
});
domainTotals.set(entry.domain, report.totals);
reports.push({ key, data: JSON.stringify(report) });
const scope = { kind: 'domain' as const, domain: entry.domain };
const counts = domain?.counts ?? emptyCounts();
const summary = buildDomainSummary(scope, counts, links, period);
domains.push({ domain: entry.domain, totals: summary.totals });
reports.push({ summary, details: buildDetails(scope, counts, period, days) });
}
const overviewSource = indexed.get('overview');
const overview = overviewSource ? scopeReport(overviewSource, period) : { ...empty };
overview.domainCounts = inventory.map((entry) => ({
domain: entry.domain,
counts: domainTotals.get(entry.domain) ?? zeroVisitCounts()
}));
reports.push({ key: 'overview', data: JSON.stringify(overview) });
const summary = buildOverviewSummary(indexed.overview, domains, period);
reports.push({
summary,
details: buildDetails(summary.scope, indexed.overview, period, days)
});
return reports;
}
function indexCounts(aggregates: AggregatedVisitReports): Map<string, ScopeCounts> {
const indexed = new Map<string, ScopeCounts>();
function scope(key: string): ScopeCounts {
let counts = indexed.get(key);
if (!counts) {
counts = { daily: [], browsers: [], operatingSystems: [] };
indexed.set(key, counts);
}
return counts;
}
for (const row of aggregates.daily) scope(row.scope).daily.push(row);
for (const row of aggregates.browsers) scope(row.scope).browsers.push(row);
for (const row of aggregates.operatingSystems) scope(row.scope).operatingSystems.push(row);
function indexCounts(aggregates: AggregatedVisitReports): ReportCounts {
const indexed: ReportCounts = { overview: emptyCounts(), domains: new Map() };
for (const row of aggregates.daily) countsForScope(indexed, row).daily.push(row);
for (const row of aggregates.browsers) countsForScope(indexed, row).browsers.push(row);
for (const row of aggregates.operatingSystems)
countsForScope(indexed, row).operatingSystems.push(row);
return indexed;
}
function emptyReport(period: VisitReportPeriod): PreparedVisitReport {
function countsForScope(
indexed: ReportCounts,
scope: DailyVisitCounts | ScopedFamilyCounts
): ScopeCounts {
if (scope.scopeKind === 'overview') return indexed.overview;
let domain = indexed.domains.get(scope.domain);
if (!domain) {
domain = { counts: emptyCounts(), links: new Map() };
indexed.domains.set(scope.domain, domain);
}
if (scope.scopeKind === 'domain') return domain.counts;
let link = domain.links.get(scope.path);
if (!link) {
link = emptyCounts();
domain.links.set(scope.path, link);
}
return link;
}
function emptyCounts(): ScopeCounts {
return { daily: [], browsers: [], operatingSystems: [] };
}
function buildOverviewSummary(
counts: ScopeCounts,
domains: OverviewSummary['domains'],
period: ReportPeriod
): OverviewSummary {
return {
totals: zeroVisitCounts(),
daily: period.days.map(() => zeroVisitCounts()),
weekly: period.weeks.map(() => zeroVisitCounts()),
monthly: period.months.map(() => zeroVisitCounts()),
browsers: [],
operatingSystems: [],
domainCounts: [],
linkCounts: []
version: 1,
scope: { kind: 'overview' },
period,
...summarizeDaily(counts.daily),
domains
};
}
function scopeReport(source: ScopeCounts, period: VisitReportPeriod): PreparedVisitReport {
const report = emptyReport(period);
const dayIndexes = new Map<number, number>();
period.days.forEach((bucket, index) => dayIndexes.set(bucket.start, index));
for (const row of source.daily) {
const start = Date.parse(`${row.day}T00:00:00Z`);
const index = dayIndexes.get(start);
if (index === undefined) continue;
addCounts(report.daily[index]!, row);
addCounts(report.totals, row);
function buildDomainSummary(
scope: DomainSummary['scope'],
counts: ScopeCounts,
links: DomainSummary['links'],
period: ReportPeriod
): DomainSummary {
return { version: 1, scope, period, ...summarizeDaily(counts.daily), links };
}
function buildLinkSummary(
scope: LinkSummary['scope'],
counts: ScopeCounts,
period: ReportPeriod
): LinkSummary {
const { totals } = summarizeDaily(counts.daily);
return { version: 1, scope, period, totals };
}
function summarizeDaily(rows: DailyVisitCounts[]): { totals: VisitTotals; base: VisitTotals } {
const totals = zeroVisitTotals();
const base = zeroVisitTotals();
for (const row of rows) {
totals.validVisits += row.validVisits;
totals.invalidVisits += row.invalidVisits;
base.validVisits += row.baseValidVisits;
base.invalidVisits += row.baseInvalidVisits;
}
for (let index = 0; index < period.days.length; index++) {
const day = period.days[index]!;
const counts = report.daily[index]!;
const week = period.weeks.findIndex(
(bucket) => day.start >= bucket.start && day.start < bucket.end
);
const month = period.months.findIndex(
(bucket) => day.start >= bucket.start && day.start < bucket.end
);
if (week >= 0) addCounts(report.weekly[week]!, counts);
if (month >= 0) addCounts(report.monthly[month]!, counts);
}
report.browsers = source.browsers.map((row) => ({ name: row.name, counts: copyCounts(row) }));
report.operatingSystems = source.operatingSystems.map((row) => ({
name: row.name,
counts: copyCounts(row)
return { totals, base };
}
function buildDetails(
scope: ReportScope,
counts: ScopeCounts,
period: ReportPeriod,
days: ReturnType<typeof visitDays>
): VisitDetails {
return {
version: 1,
scope,
period,
daily: buildDailyDetails(counts.daily, days),
browsers: buildFamilyDetails(counts.browsers),
operatingSystems: buildFamilyDetails(counts.operatingSystems)
};
}
function buildDailyDetails(
rows: DailyVisitCounts[],
days: ReturnType<typeof visitDays>
): VisitDetails['daily'] {
const countsByDay = new Map(rows.map((row) => [row.day, row]));
return days.map((day) => {
const counts = countsByDay.get(day.start.slice(0, 10));
return {
...day,
validVisits: counts?.validVisits ?? 0,
invalidVisits: counts?.invalidVisits ?? 0
};
});
}
function buildFamilyDetails(rows: ScopedFamilyCounts[]): VisitDetails['browsers'] {
return rows.map(({ family, validVisits, invalidVisits }) => ({
family,
validVisits,
invalidVisits
}));
return report;
}
function copyCounts(counts: VisitCounts): VisitCounts {
return { total: counts.total, valid: counts.valid, invalid: counts.invalid, base: counts.base };
}
function addCounts(target: VisitCounts, counts: VisitCounts): void {
target.total += counts.total;
target.valid += counts.valid;
target.invalid += counts.invalid;
target.base += counts.base;
}
@@ -1,6 +1,7 @@
import { encodeVisitReports, type VisitTotals } from '@goto/analytics';
import { validateConfig } from '@goto/config';
import { openDb, runMigrations, insertVisit, pruneVisits, type ClassifiedVisit } from '@goto/db';
import { readPreparedVisitReports } from '@goto/db/visit-reports';
import { readVisitSummary, readVisitDetails } from '@goto/db/visit-reports';
import { afterEach, expect, test, vi } from 'vitest';
import { prepareVisitReports } from './prepare-visit-reports.js';
import type { WorkerContext } from './context.js';
@@ -10,7 +11,15 @@ vi.mock('@goto/db', async (importOriginal) => {
return { ...actual, pruneVisits: vi.fn(actual.pruneVisits) };
});
afterEach(() => vi.mocked(pruneVisits).mockReset());
vi.mock('@goto/analytics', async (importOriginal) => {
const actual = await importOriginal<typeof import('@goto/analytics')>();
return { ...actual, encodeVisitReports: vi.fn(actual.encodeVisitReports) };
});
afterEach(() => {
vi.mocked(pruneVisits).mockReset();
vi.mocked(encodeVisitReports).mockReset();
});
async function context(): Promise<WorkerContext> {
const prepared = validateConfig(
@@ -35,12 +44,21 @@ const visit: ClassifiedVisit = {
browser: 'Firefox',
os: 'Linux'
};
const overview = { kind: 'overview' as const };
const domain = { kind: 'domain' as const, domain: 'go.example.test' };
const docs = { kind: 'link' as const, domain: 'go.example.test', path: '/docs' };
function sum(values: { total: number }[]): number {
return values.reduce((total, value) => total + value.total, 0);
function sum(values: VisitTotals[]): VisitTotals {
return values.reduce(
(total, value) => ({
validVisits: total.validVisits + value.validVisits,
invalidVisits: total.invalidVisits + value.invalidVisits
}),
{ validVisits: 0, invalidVisits: 0 }
);
}
test('one refresh publishes all scopes and chart resolutions, excludes/deletes expired data and is repeatable', async () => {
test('publishes summary/detail pairs for all scopes, deletes expired history and is repeatable', async () => {
const ctx = await context();
try {
const now = Date.parse('2026-05-31T12:00:00Z');
@@ -52,18 +70,27 @@ test('one refresh publishes all scopes and chart resolutions, excludes/deletes e
await insertVisit(ctx.db, { ...visit, visited_at: cutoff });
await insertVisit(ctx.db, { ...visit, path: '/', outcome: 'invalid', visited_at: now - 2000 });
await prepareVisitReports(ctx, now);
const reports = await readPreparedVisitReports(ctx.db, 'go.example.test', '/docs');
expect(reports?.report.totals).toEqual({ total: 2, valid: 1, invalid: 1, base: 1 });
expect(reports?.selection.totals.total).toBe(1);
expect(sum(reports!.report.daily)).toBe(2);
expect(sum(reports!.report.weekly)).toBe(2);
expect(sum(reports!.report.monthly)).toBe(2);
expect(reports?.report.browsers).toEqual([
{ name: 'Firefox', counts: { total: 2, valid: 1, invalid: 1, base: 1 } }
const summary = await readVisitSummary(ctx.db, domain);
const details = await readVisitDetails(ctx.db, domain);
expect(summary?.totals).toEqual({ validVisits: 1, invalidVisits: 1 });
expect(summary?.scope).toEqual(domain);
expect(summary?.period.from).toBe('2026-02-28T12:00:00.000Z');
expect(summary?.period.asOf).toBe('2026-05-31T12:00:00.000Z');
expect(details?.scope).toEqual(summary?.scope);
expect(details?.period).toEqual(summary?.period);
expect((await readVisitSummary(ctx.db, docs))?.totals).toEqual({
validVisits: 1,
invalidVisits: 0
});
expect(sum(details!.daily)).toEqual(summary?.totals);
expect(details?.browsers).toEqual([{ family: 'Firefox', validVisits: 1, invalidVisits: 1 }]);
expect(details?.operatingSystems).toEqual([
{ family: 'Linux', validVisits: 1, invalidVisits: 1 }
]);
expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toHaveLength(4);
expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(2);
await prepareVisitReports(ctx, now);
expect((await readPreparedVisitReports(ctx.db))?.report.totals.total).toBe(2);
expect((await readVisitSummary(ctx.db, overview))?.totals).toEqual(summary?.totals);
} finally {
await ctx.db.destroy();
}
@@ -78,7 +105,31 @@ test('zero purges raw history and prepared output without leaving a longer-lived
ctx.config.analytics.retention_months = 0;
await prepareVisitReports(ctx, now);
expect(await ctx.db.selectFrom('visits').select('id').execute()).toEqual([]);
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
} finally {
await ctx.db.destroy();
}
});
test('encodes the whole batch before clearing output or pruning history', async () => {
const ctx = await context();
try {
const now = Date.parse('2026-05-31T12:00:00Z');
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
await prepareVisitReports(ctx, now);
const previous = await readVisitSummary(ctx.db, overview);
vi.mocked(pruneVisits).mockClear();
const failure = new Error('Invalid report pair');
vi.mocked(encodeVisitReports).mockClear();
vi.mocked(encodeVisitReports).mockImplementationOnce(() => {
throw failure;
});
await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure);
expect(encodeVisitReports).toHaveBeenCalledOnce();
expect(pruneVisits).not.toHaveBeenCalled();
expect(await readVisitSummary(ctx.db, overview)).toEqual(previous);
expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(1);
} finally {
await ctx.db.destroy();
}
@@ -89,16 +140,17 @@ test('clears old output before pruning and publishes nothing if pruning fails',
try {
const now = Date.parse('2026-05-31T12:00:00Z');
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
await prepareVisitReports(ctx, now);
vi.mocked(pruneVisits).mockClear();
const failure = new Error('Pruning failed');
vi.mocked(pruneVisits).mockImplementationOnce(async () => {
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
throw failure;
});
await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure);
expect(pruneVisits).toHaveBeenCalledExactlyOnceWith(ctx.db, Date.parse('2026-02-28T12:00:00Z'));
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
} finally {
await ctx.db.destroy();
}
@@ -109,7 +161,8 @@ test('stops pruning and does not publish obsolete output when positive retention
try {
const now = Date.parse('2026-05-31T12:00:00Z');
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
await prepareVisitReports(ctx, now);
vi.mocked(pruneVisits).mockClear();
vi.mocked(pruneVisits).mockImplementationOnce(async () => {
ctx.config.analytics.retention_months = 9;
return true;
@@ -117,7 +170,7 @@ test('stops pruning and does not publish obsolete output when positive retention
await prepareVisitReports(ctx, now);
expect(pruneVisits).toHaveBeenCalledOnce();
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
expect(await ctx.db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
} finally {
await ctx.db.destroy();
}
@@ -1,4 +1,5 @@
import { setImmediate } from 'node:timers/promises';
import { createVisitPeriod, encodeVisitReports } from '@goto/analytics';
import { pruneVisits, purgeVisits } from '@goto/db';
import {
aggregateVisitReports,
@@ -7,7 +8,6 @@ import {
type ReportInventory
} from '@goto/db/visit-reports';
import type { WorkerContext } from './context.js';
import { reportPeriod } from './report-buckets.js';
import { buildVisitReports } from './build-visit-reports.js';
export async function prepareVisitReports(
@@ -27,18 +27,20 @@ export async function prepareVisitReports(
domain,
paths: Object.keys(prepared.links)
}));
const period = reportPeriod(months, asOf);
const counts = await aggregateVisitReports(context.db, period.since, asOf, inventory);
const period = createVisitPeriod(months, asOf);
const since = Date.parse(period.from);
const counts = await aggregateVisitReports(context.db, since, asOf, inventory);
const reports = buildVisitReports(counts, inventory, period);
const rows = encodeVisitReports(reports);
if (context.config.analytics.retention_months !== months) return;
// Retire old output before deleting any history represented by it.
await clearVisitReports(context.db);
while (context.config.analytics.retention_months === months) {
const more = await pruneVisits(context.db, period.since);
const more = await pruneVisits(context.db, since);
if (!more) break;
await setImmediate();
}
if (context.config.analytics.retention_months !== months) return;
await publishVisitReports(context.db, period, reports);
await publishVisitReports(context.db, rows);
}
@@ -1,33 +0,0 @@
import { expect, test } from 'vitest';
import { reportPeriod } from './report-buckets.js';
test('prepares UTC days, Monday weeks and calendar months with honest partial edges', () => {
const asOf = Date.parse('2026-05-31T12:34:56.789Z');
const period = reportPeriod(3, asOf);
expect(period.since).toBe(Date.parse('2026-02-28T12:34:56.789Z'));
expect(period.asOf).toBe(asOf);
expect(period.days[0]).toEqual({
start: Date.parse('2026-02-28T00:00:00Z'),
end: Date.parse('2026-03-01T00:00:00Z'),
partial: true
});
expect(period.days[1]).toEqual({
start: Date.parse('2026-03-01T00:00:00Z'),
end: Date.parse('2026-03-02T00:00:00Z'),
partial: false
});
expect(period.weeks[1]).toEqual({
start: Date.parse('2026-03-02T00:00:00Z'),
end: Date.parse('2026-03-09T00:00:00Z'),
partial: false
});
expect(period.months[1]).toEqual({
start: Date.parse('2026-03-01T00:00:00Z'),
end: Date.parse('2026-04-01T00:00:00Z'),
partial: false
});
expect(new Date(period.weeks[0]!.start).getUTCDay()).toBe(1);
expect(period.months[0]!.start).toBe(Date.parse('2026-02-01T00:00:00Z'));
expect(period.days.at(-1)!.partial).toBe(true);
expect(period.months.at(-1)!.partial).toBe(true);
});
@@ -1,36 +0,0 @@
import type { TimeBucket, VisitReportPeriod } from '@goto/db/visit-reports';
import { retentionCutoff, type RetentionMonths } from './retention.js';
type BucketSize = 'day' | 'week' | 'month';
export function reportPeriod(months: Exclude<RetentionMonths, 0>, asOf: number): VisitReportPeriod {
const since = retentionCutoff(months, asOf);
return {
timezone: 'UTC',
retentionMonths: months,
since,
asOf,
days: timeBuckets(since, asOf, 'day'),
weeks: timeBuckets(since, asOf, 'week'),
months: timeBuckets(since, asOf, 'month')
};
}
function timeBuckets(since: number, asOf: number, size: BucketSize): TimeBucket[] {
const date = new Date(since);
date.setUTCHours(0, 0, 0, 0);
if (size === 'week') {
const daysSinceMonday = (date.getUTCDay() + 6) % 7;
date.setUTCDate(date.getUTCDate() - daysSinceMonday);
}
if (size === 'month') date.setUTCDate(1);
const buckets: TimeBucket[] = [];
while (date.getTime() < asOf) {
const start = date.getTime();
if (size === 'month') date.setUTCMonth(date.getUTCMonth() + 1);
else date.setUTCDate(date.getUTCDate() + (size === 'week' ? 7 : 1));
const end = date.getTime();
buckets.push({ start, end, partial: start < since || end > asOf });
}
return buckets;
}
@@ -1,8 +1,11 @@
import { createVisitPeriod, encodeVisitReports } from '@goto/analytics';
import { validateConfig } from '@goto/config';
import { openDb, runMigrations } from '@goto/db';
import { publishVisitReports } from '@goto/db/visit-reports';
import type { OpenWorkflow } from 'openworkflow';
import { expect, test, vi } from 'vitest';
import type { WorkerContext } from './context.js';
import { buildVisitReports } from './build-visit-reports.js';
import { configureReportSchedule, requestReportRefresh } from './report-schedule.js';
import { refreshVisitReports, reportSchedule } from '../workflows/specs.js';
@@ -46,7 +49,13 @@ test('zero pauses hourly scheduling and purges prepared output even if raw histo
expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(reportSchedule.id);
await requestReportRefresh(ctx, sdk);
expect(engine.runWorkflow).not.toHaveBeenCalled();
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
const period = createVisitPeriod(3, Date.now());
const reports = buildVisitReports(
{ daily: [], browsers: [], operatingSystems: [] },
[],
period
);
await publishVisitReports(ctx.db, encodeVisitReports(reports));
await requestReportRefresh(ctx, sdk);
expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(refreshVisitReports, {});
} finally {
+2 -2
View File
@@ -19,8 +19,8 @@ export async function requestReportRefresh(
if (context.config.analytics.retention_months === 0) {
const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst();
const output = await context.db
.selectFrom('visit_reports')
.select('key')
.selectFrom('analytics_reports')
.select('id')
.limit(1)
.executeTakeFirst();
if (!visit && !output) return;
@@ -1,18 +0,0 @@
import { expect, test } from 'vitest';
import { retentionCutoff } from './retention.js';
test('uses UTC calendar months, clamping the day while preserving the time', () => {
const now = Date.parse('2026-05-31T12:34:56.789Z');
expect(new Date(retentionCutoff(3, now)).toISOString()).toBe('2026-02-28T12:34:56.789Z');
expect(new Date(retentionCutoff(9, now)).toISOString()).toBe('2025-08-31T12:34:56.789Z');
expect(new Date(retentionCutoff(12, now)).toISOString()).toBe('2025-05-31T12:34:56.789Z');
});
test('handles leap-day boundaries without overflowing into March', () => {
expect(new Date(retentionCutoff(3, Date.parse('2024-05-31T01:00:00Z'))).toISOString()).toBe(
'2024-02-29T01:00:00.000Z'
);
expect(new Date(retentionCutoff(12, Date.parse('2024-02-29T01:00:00Z'))).toISOString()).toBe(
'2023-02-28T01:00:00.000Z'
);
});
-15
View File
@@ -1,15 +0,0 @@
import type { GotoConfiguration } from '@goto/config';
export type RetentionMonths = GotoConfiguration['analytics']['retention_months'];
export function retentionCutoff(months: Exclude<RetentionMonths, 0>, now = Date.now()): number {
const cutoff = new Date(now);
const day = cutoff.getUTCDate();
cutoff.setUTCDate(1);
cutoff.setUTCMonth(cutoff.getUTCMonth() - months);
const lastDay = new Date(
Date.UTC(cutoff.getUTCFullYear(), cutoff.getUTCMonth() + 1, 0)
).getUTCDate();
cutoff.setUTCDate(Math.min(day, lastDay));
return cutoff.getTime();
}
+12 -1
View File
@@ -3,10 +3,13 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import { createVisitPeriod, encodeVisitReports } from '@goto/analytics';
import { insertVisit, openDb, runMigrations } from '@goto/db';
import { publishVisitReports } from '@goto/db/visit-reports';
import { Worker } from 'openworkflow';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import { startWorkerService } from './startup.js';
import { buildVisitReports } from './build-visit-reports.js';
let directory: string;
@@ -61,12 +64,19 @@ test('starts an idle zero-policy service, migrates visit storage, and exposes he
}
});
test('an existing zero-policy database is purged by a real startup workflow', async () => {
test('a real zero-policy startup workflow purges raw history and prepared output', async () => {
await writeConfiguration();
const path = join(directory, 'data/visits.sqlite');
const db = await openDb(path);
try {
await runMigrations(db);
const period = createVisitPeriod(3, Date.now() - 1000);
const reports = buildVisitReports(
{ daily: [], browsers: [], operatingSystems: [] },
[],
period
);
await publishVisitReports(db, encodeVisitReports(reports));
await insertVisit(db, {
domain: 'go.example.test',
path: '/docs',
@@ -86,6 +96,7 @@ test('an existing zero-policy database is purged by a real startup workflow', as
const reader = await openDb(path, { readOnly: true });
try {
expect(await reader.selectFrom('visits').selectAll().execute()).toEqual([]);
expect(await reader.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
} finally {
await reader.destroy();
}
+34
View File
@@ -0,0 +1,34 @@
{
"name": "@goto/analytics",
"version": "0.0.0",
"private": true,
"type": "module",
"exports": {
".": {
"development": "./src/index.ts",
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"files": [
"dist"
],
"scripts": {
"build": "tsc -p tsconfig.build.json",
"check": "tsc --noEmit -p tsconfig.json",
"lint": "prettier --ignore-path ../../.prettierignore --check . && eslint .",
"format": "prettier --ignore-path ../../.prettierignore --write .",
"test": "vitest run",
"test:coverage": "vitest run --coverage"
},
"dependencies": {
"@sinclair/typebox": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"@vitest/coverage-v8": "catalog:",
"prettier": "catalog:",
"typescript": "catalog:",
"vitest": "catalog:"
}
}
+102
View File
@@ -0,0 +1,102 @@
import { expect, test } from 'vitest';
import { parseVisitDetails, parseVisitSummary } from './decode.js';
import { createVisitPeriod, visitDays } from './period.js';
import { zeroVisitTotals, type OverviewSummary, type VisitDetails } from './reports.js';
const scope = { kind: 'overview' } as const;
const period = createVisitPeriod(3, Date.UTC(2026, 3, 1, 12));
const summary: OverviewSummary = {
version: 1,
scope,
period,
totals: { validVisits: 2, invalidVisits: 1 },
base: { validVisits: 1, invalidVisits: 0 },
domains: []
};
const details: VisitDetails = {
version: 1,
scope,
period,
daily: visitDays(period).map((day, index) => ({
...day,
...(index === 0 ? summary.totals : zeroVisitTotals())
})),
browsers: [{ family: 'Firefox', ...summary.totals }],
operatingSystems: [{ family: 'Linux', ...summary.totals }]
};
test('readers decode their self-contained document independently', () => {
expect(parseVisitSummary(JSON.stringify(summary), scope)).toEqual(summary);
expect(parseVisitDetails(JSON.stringify(details), scope)).toEqual(details);
});
test('summary reader rejects unsupported versions, unsafe counts and wrong identities safely', () => {
for (const invalid of [
{ ...summary, version: 2 },
{ ...summary, totals: { validVisits: -1, invalidVisits: 0 } },
{ ...summary, totals: { validVisits: 0.5, invalidVisits: 0 } },
{ ...summary, totals: { validVisits: Number.MAX_SAFE_INTEGER + 1, invalidVisits: 0 } },
{ ...summary, unexpected: 'private' }
]) {
expect(() => parseVisitSummary(JSON.stringify(invalid), scope)).toThrow(
'Invalid analytics report'
);
}
expect(() =>
parseVisitSummary(JSON.stringify(summary), { kind: 'domain', domain: 'private' })
).toThrow('Invalid analytics report');
expect(() => parseVisitSummary('{private', scope)).toThrow('Invalid analytics report');
});
test('readers reject noncanonical, reversed and non-retention windows', () => {
for (const invalidPeriod of [
{ ...period, asOf: 'not-a-date' },
{ ...period, asOf: period.asOf.replace('.000Z', 'Z') },
{ ...period, from: '2026-02-30T12:00:00.000Z' },
{ ...period, from: period.asOf },
{ ...period, from: '2025-01-01T00:00:00.000Z' }
]) {
expect(() =>
parseVisitSummary(JSON.stringify({ ...summary, period: invalidPeriod }), scope)
).toThrow('Invalid analytics report');
expect(() =>
parseVisitDetails(JSON.stringify({ ...details, period: invalidPeriod }), scope)
).toThrow('Invalid analytics report');
}
});
test('details reader checks daily instants, window safety, counts and scope', () => {
const first = details.daily[0];
if (!first) throw new Error('Expected a daily fixture');
for (const point of [
{ ...first, start: 'not-a-date' },
{ ...first, end: first.end.replace('.000Z', 'Z') },
{ ...first, start: '2026-02-30T00:00:00.000Z' },
{ ...first, start: first.end },
{ ...first, start: period.asOf, end: '2026-04-02T00:00:00.000Z' },
{ ...first, invalidVisits: -1 }
]) {
expect(() => parseVisitDetails(JSON.stringify({ ...details, daily: [point] }), scope)).toThrow(
'Invalid analytics report'
);
}
expect(() => parseVisitDetails(JSON.stringify({ ...details, version: 2 }), scope)).toThrow(
'Invalid analytics report'
);
expect(() =>
parseVisitDetails(JSON.stringify(details), { kind: 'domain', domain: 'private' })
).toThrow('Invalid analytics report');
expect(() => parseVisitDetails('{private', scope)).toThrow('Invalid analytics report');
});
test('reader prevents a base subset from exceeding totals used by page subtraction', () => {
expect(() =>
parseVisitSummary(
JSON.stringify({
...summary,
base: { validVisits: 3, invalidVisits: 0 }
}),
scope
)
).toThrow('Invalid analytics report');
});
+24
View File
@@ -0,0 +1,24 @@
import type { ReportScope, VisitDetails, VisitSummary } from './reports.js';
import { checkDetails, checkSummary, invalidReport, sameScope } from './validation.js';
export function parseVisitSummary(json: string, expectedScope: ReportScope): VisitSummary {
const document = parseDocument(json);
const summary = checkSummary(document);
if (!sameScope(summary.scope, expectedScope)) throw invalidReport();
return summary;
}
export function parseVisitDetails(json: string, expectedScope: ReportScope): VisitDetails {
const document = parseDocument(json);
const details = checkDetails(document);
if (!sameScope(details.scope, expectedScope)) throw invalidReport();
return details;
}
function parseDocument(json: string): unknown {
try {
return JSON.parse(json);
} catch {
throw invalidReport();
}
}
+200
View File
@@ -0,0 +1,200 @@
import { expect, test, vi } from 'vitest';
import { parseVisitDetails, parseVisitSummary } from './decode.js';
import { encodeVisitReports } from './encode.js';
import { createVisitPeriod, visitDays } from './period.js';
import {
zeroVisitTotals,
type PreparedVisitReport,
type ReportScope,
type VisitDetails,
type VisitSummary,
type VisitTotals
} from './reports.js';
const overview = { kind: 'overview' } as const;
const domain = { kind: 'domain', domain: 'go.example.test' } as const;
const root = { kind: 'link', domain: domain.domain, path: '/' } as const;
const asOf = Date.UTC(2026, 3, 1, 12);
function report(
scope: ReportScope = overview,
cutoff = asOf,
totals: VisitTotals = { validVisits: 2, invalidVisits: 1 }
): PreparedVisitReport {
const period = createVisitPeriod(3, cutoff);
let summary: VisitSummary;
if (scope.kind === 'overview') {
summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), domains: [] };
} else if (scope.kind === 'domain') {
summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), links: [] };
} else {
summary = { version: 1, scope, period, totals };
}
const populated = totals.validVisits > 0 || totals.invalidVisits > 0;
return {
summary,
details: {
version: 1,
scope,
period,
daily: visitDays(period).map((day, index) => ({
...day,
...(index === 0 ? totals : zeroVisitTotals())
})),
browsers: populated ? [{ family: 'Firefox', ...totals }] : [],
operatingSystems: populated ? [{ family: 'Linux', ...totals }] : []
}
};
}
test('encodes each summary variant and explicit root without changing document shapes', () => {
const reports = [report(overview), report(domain), report(root, asOf, zeroVisitTotals())];
const rows = encodeVisitReports(reports);
for (const [index, prepared] of reports.entries()) {
const row = rows[index];
if (!row) throw new Error('Expected a serialized fixture');
expect(row.scope).toEqual(prepared.summary.scope);
expect(parseVisitSummary(row.summaryJson, row.scope)).toEqual(prepared.summary);
expect(parseVisitDetails(row.detailsJson, row.scope)).toEqual(prepared.details);
}
expect(encodeVisitReports([])).toEqual([]);
});
test('encodes each document once without a stringify-to-parse validation loop', () => {
const reports = [report(overview), report(domain)];
const stringify = vi.spyOn(JSON, 'stringify');
const parse = vi.spyOn(JSON, 'parse');
try {
encodeVisitReports(reports);
expect(stringify).toHaveBeenCalledTimes(4);
expect(parse).not.toHaveBeenCalled();
} finally {
stringify.mockRestore();
parse.mockRestore();
}
});
test('rejects incompatible pair identities, periods, totals and document versions', () => {
const prepared = report();
for (const invalid of [
{ ...prepared, summary: { ...prepared.summary, totals: { validVisits: 3, invalidVisits: 1 } } },
{ ...prepared, details: { ...prepared.details, scope: root } },
{
...prepared,
details: { ...prepared.details, period: createVisitPeriod(3, asOf + 86_400_000) }
}
]) {
expect(() => encodeVisitReports([invalid])).toThrow('Invalid analytics report');
}
const wrongVersion = report();
Object.assign(wrongVersion.details, { version: 2 });
expect(() => encodeVisitReports([wrongVersion])).toThrow('Invalid analytics report');
});
test('batch checks duplicate scope identities and shared periods before serialization', () => {
const stringify = vi.spyOn(JSON, 'stringify');
try {
for (const scope of [overview, domain, root]) {
expect(() => encodeVisitReports([report(scope), report(scope)])).toThrow(
'Invalid analytics report'
);
}
expect(() => encodeVisitReports([report(overview), report(domain, asOf + 86_400_000)])).toThrow(
'Invalid analytics report'
);
expect(stringify).not.toHaveBeenCalled();
} finally {
stringify.mockRestore();
}
});
test('producer requires gap-free UTC days and correct partial flags including zero points', () => {
const prepared = report();
const daily = prepared.details.daily;
const first = daily[0];
if (!first) throw new Error('Expected a daily fixture');
const invalidDaily: VisitDetails['daily'][] = [
daily.slice(1),
[...daily].reverse(),
[{ ...first, start: prepared.details.period.from }, ...daily.slice(1)],
[{ ...first, end: prepared.details.period.asOf }, ...daily.slice(1)],
[{ ...first, partial: false }, ...daily.slice(1)]
];
for (const points of invalidDaily) {
expect(() =>
encodeVisitReports([
{
...prepared,
details: { ...prepared.details, daily: points }
}
])
).toThrow('Invalid analytics report');
}
});
test('producer checks family sums and duplicate labels against the daily totals', () => {
const prepared = report();
for (const details of [
{ ...prepared.details, browsers: [] },
{
...prepared.details,
operatingSystems: [{ family: 'Linux', validVisits: 3, invalidVisits: 0 }]
},
{
...prepared.details,
browsers: [
{ family: 'Firefox', validVisits: 1, invalidVisits: 0 },
{ family: 'Firefox', validVisits: 1, invalidVisits: 1 }
]
}
]) {
expect(() => encodeVisitReports([{ ...prepared, details }])).toThrow(
'Invalid analytics report'
);
}
});
test('producer rejects base or compact child subsets above totals and duplicate children', () => {
const prepared = report();
for (const summary of [
{ ...prepared.summary, base: { validVisits: 3, invalidVisits: 0 } },
{
...prepared.summary,
domains: [
{ domain: domain.domain, totals: zeroVisitTotals() },
{ domain: domain.domain, totals: zeroVisitTotals() }
]
},
{
...prepared.summary,
domains: [{ domain: domain.domain, totals: { validVisits: 5, invalidVisits: 0 } }]
}
]) {
expect(() => encodeVisitReports([{ ...prepared, summary }])).toThrow(
'Invalid analytics report'
);
}
const domainReport = report(domain);
const duplicateLinks = {
...domainReport.summary,
links: [
{ path: '/', totals: zeroVisitTotals() },
{ path: '/', totals: zeroVisitTotals() }
]
};
expect(() => encodeVisitReports([{ ...domainReport, summary: duplicateLinks }])).toThrow(
'Invalid analytics report'
);
});
test('producer rejects unsafe summed counts even when each input count is safe', () => {
const prepared = report(overview, asOf, {
validVisits: Number.MAX_SAFE_INTEGER,
invalidVisits: 0
});
const secondDay = prepared.details.daily[1];
if (!secondDay) throw new Error('Expected a second daily fixture');
secondDay.validVisits = 1;
expect(() => encodeVisitReports([prepared])).toThrow('Invalid analytics report');
});
+119
View File
@@ -0,0 +1,119 @@
import { visitDays } from './period.js';
import {
zeroVisitTotals,
type PreparedVisitReport,
type SerializedVisitReport,
type VisitDetails,
type VisitSummary,
type VisitTotals
} from './reports.js';
import {
checkDetails,
checkSummary,
equalTotals,
invalidReport,
samePeriod,
sameScope,
withinTotals
} from './validation.js';
export function encodeVisitReports(reports: PreparedVisitReport[]): SerializedVisitReport[] {
const first = reports[0];
if (!first) return [];
let overview = false;
const domains = new Set<string>();
const links = new Map<string, Set<string>>();
for (const report of reports) {
const summary = checkSummary(report.summary);
const details = checkDetails(report.details);
if (
!sameScope(summary.scope, details.scope) ||
!samePeriod(summary.period, details.period) ||
!samePeriod(summary.period, first.summary.period)
) {
throw invalidReport();
}
const scope = summary.scope;
if (scope.kind === 'overview') {
if (overview) throw invalidReport();
overview = true;
} else if (scope.kind === 'domain') {
if (domains.has(scope.domain)) throw invalidReport();
domains.add(scope.domain);
} else {
const paths = links.get(scope.domain) ?? new Set<string>();
if (paths.has(scope.path)) throw invalidReport();
paths.add(scope.path);
links.set(scope.domain, paths);
}
}
const days = visitDays(first.summary.period);
for (const { summary, details } of reports) {
checkDailyLayout(details.daily, days);
const totals = sumTotals(details.daily);
if (!equalTotals(summary.totals, totals)) throw invalidReport();
checkFamilies(details.browsers, totals);
checkFamilies(details.operatingSystems, totals);
checkChildren(summary);
}
return reports.map(({ summary, details }) => ({
scope: summary.scope,
summaryJson: JSON.stringify(summary),
detailsJson: JSON.stringify(details)
}));
}
function checkDailyLayout(daily: VisitDetails['daily'], days: ReturnType<typeof visitDays>): void {
if (daily.length !== days.length) throw invalidReport();
for (const [index, day] of days.entries()) {
const point = daily[index];
if (
!point ||
point.start !== day.start ||
point.end !== day.end ||
point.partial !== day.partial
) {
throw invalidReport();
}
}
}
function checkFamilies(rows: VisitDetails['browsers'], totals: VisitTotals): void {
const families = new Set(rows.map(({ family }) => family));
const combined = sumTotals(rows);
if (families.size !== rows.length || !equalTotals(combined, totals)) {
throw invalidReport();
}
}
function checkChildren(summary: VisitSummary): void {
if ('domains' in summary) {
if (new Set(summary.domains.map(({ domain }) => domain)).size !== summary.domains.length) {
throw invalidReport();
}
const childTotals = summary.domains.map(({ totals }) => totals);
const combined = sumTotals(childTotals);
if (!withinTotals(combined, summary.totals)) throw invalidReport();
} else if ('links' in summary) {
if (new Set(summary.links.map(({ path }) => path)).size !== summary.links.length) {
throw invalidReport();
}
const childTotals = summary.links.map(({ totals }) => totals);
const combined = sumTotals(childTotals);
if (!withinTotals(combined, summary.totals)) throw invalidReport();
}
}
function sumTotals(rows: VisitTotals[]): VisitTotals {
const totals = zeroVisitTotals();
for (const row of rows) {
totals.validVisits += row.validVisits;
totals.invalidVisits += row.invalidVisits;
}
if (!Number.isSafeInteger(totals.validVisits) || !Number.isSafeInteger(totals.invalidVisits)) {
throw invalidReport();
}
return totals;
}
+16
View File
@@ -0,0 +1,16 @@
export { createVisitPeriod, visitDays } from './period.js';
export { zeroVisitTotals } from './reports.js';
export { encodeVisitReports } from './encode.js';
export { parseVisitSummary, parseVisitDetails } from './decode.js';
export type {
ReportScope,
ReportPeriod,
VisitTotals,
VisitSummary,
OverviewSummary,
DomainSummary,
LinkSummary,
VisitDetails,
PreparedVisitReport,
SerializedVisitReport
} from './reports.js';
+51
View File
@@ -0,0 +1,51 @@
import { expect, test } from 'vitest';
import { createVisitPeriod, visitDays } from './period.js';
test('period subtraction clamps UTC month ends and preserves time of day', () => {
expect(createVisitPeriod(3, Date.UTC(2024, 4, 31, 12, 34, 56, 789))).toEqual({
from: '2024-02-29T12:34:56.789Z',
asOf: '2024-05-31T12:34:56.789Z',
timezone: 'UTC',
retentionMonths: 3
});
expect(createVisitPeriod(9, Date.UTC(2026, 0, 31)).from).toBe('2025-04-30T00:00:00.000Z');
expect(createVisitPeriod(12, Date.UTC(2024, 1, 29)).from).toBe('2023-02-28T00:00:00.000Z');
});
test('daily points cover the full window in UTC with partial edge days', () => {
const period = createVisitPeriod(3, Date.UTC(2026, 4, 31, 12));
const days = visitDays(period);
expect(days[0]).toEqual({
start: '2026-02-28T00:00:00.000Z',
end: '2026-03-01T00:00:00.000Z',
partial: true
});
expect(days.at(-1)).toEqual({
start: '2026-05-31T00:00:00.000Z',
end: '2026-06-01T00:00:00.000Z',
partial: true
});
expect(days.slice(1, -1).every((day) => !day.partial)).toBe(true);
for (let index = 1; index < days.length; index++) {
expect(days[index]?.start).toBe(days[index - 1]?.end);
}
});
test('midnight cutoff omits the following day and leaves full days unmarked', () => {
const days = visitDays(createVisitPeriod(3, Date.UTC(2026, 3, 1)));
expect(days).toHaveLength(90);
expect(days[0]?.start).toBe('2026-01-01T00:00:00.000Z');
expect(days.at(-1)?.end).toBe('2026-04-01T00:00:00.000Z');
expect(days.every((day) => !day.partial)).toBe(true);
});
test('period helpers reject invalid instants instead of normalizing dates silently', () => {
expect(() => createVisitPeriod(3, Number.NaN)).toThrow('Invalid analytics period');
const period = createVisitPeriod(3, Date.UTC(2026, 3, 1));
expect(() => visitDays({ ...period, from: '2026-02-30T00:00:00.000Z' })).toThrow(
'Invalid analytics period'
);
expect(() => visitDays({ ...period, from: period.asOf })).toThrow('Invalid analytics period');
});
+54
View File
@@ -0,0 +1,54 @@
import type { ReportPeriod } from './reports.js';
export function isCanonicalInstant(value: string): boolean {
const time = Date.parse(value);
return Number.isFinite(time) && new Date(time).toISOString() === value;
}
export function createVisitPeriod(months: 3 | 9 | 12, asOf: number): ReportPeriod {
if (![3, 9, 12].includes(months) || !Number.isFinite(asOf)) {
throw new Error('Invalid analytics period');
}
const end = new Date(asOf);
const start = new Date(asOf);
const day = start.getUTCDate();
start.setUTCDate(1);
start.setUTCMonth(start.getUTCMonth() - months);
const lastDay = new Date(start);
lastDay.setUTCMonth(lastDay.getUTCMonth() + 1);
lastDay.setUTCDate(0);
start.setUTCDate(Math.min(day, lastDay.getUTCDate()));
if (!Number.isFinite(start.getTime()) || !Number.isFinite(end.getTime())) {
throw new Error('Invalid analytics period');
}
return {
from: start.toISOString(),
asOf: end.toISOString(),
timezone: 'UTC',
retentionMonths: months
};
}
export function visitDays(
period: ReportPeriod
): Array<{ start: string; end: string; partial: boolean }> {
const from = Date.parse(period.from);
const asOf = Date.parse(period.asOf);
if (!isCanonicalInstant(period.from) || !isCanonicalInstant(period.asOf) || from >= asOf) {
throw new Error('Invalid analytics period');
}
const date = new Date(from);
date.setUTCHours(0, 0, 0, 0);
const days = [];
while (date.getTime() < asOf) {
const start = date.toISOString();
const startTime = date.getTime();
date.setUTCDate(date.getUTCDate() + 1);
days.push({
start,
end: date.toISOString(),
partial: startTime < from || date.getTime() > asOf
});
}
return days;
}
+37
View File
@@ -0,0 +1,37 @@
import { Value } from '@sinclair/typebox/value';
import { expect, test } from 'vitest';
import { createVisitPeriod } from './period.js';
import {
ReportScopeSchema,
VisitSummarySchema,
zeroVisitTotals,
type LinkSummary
} from './reports.js';
test('zero totals are independent objects with no redundant total or root count', () => {
const first = zeroVisitTotals();
const second = zeroVisitTotals();
first.validVisits = 1;
expect(second).toEqual({ validVisits: 0, invalidVisits: 0 });
expect(first).not.toBe(second);
});
test('canonical schemas distinguish scopes and summary variants', () => {
const summary: LinkSummary = {
version: 1,
scope: { kind: 'link', domain: 'go.example.test', path: '/' },
period: createVisitPeriod(3, Date.UTC(2026, 2, 1)),
totals: zeroVisitTotals()
};
expect(Value.Check(VisitSummarySchema, summary)).toBe(true);
expect(Value.Check(VisitSummarySchema, { ...summary, base: zeroVisitTotals() })).toBe(false);
expect(Value.Check(ReportScopeSchema, { kind: 'overview', domain: null })).toBe(false);
expect(
Value.Check(ReportScopeSchema, { kind: 'domain', domain: 'go.example.test', path: '/' })
).toBe(false);
expect(
Value.Check(ReportScopeSchema, { kind: 'link', domain: 'go.example.test', path: '' })
).toBe(false);
});
+123
View File
@@ -0,0 +1,123 @@
import { FormatRegistry, Type, type Static } from '@sinclair/typebox';
import { isCanonicalInstant } from './period.js';
const exact = { additionalProperties: false };
const count = Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER });
const domain = Type.String({ minLength: 1 });
const path = Type.String({ pattern: '^/' });
FormatRegistry.Set('goto-analytics-utc-instant', isCanonicalInstant);
const instant = Type.String({ format: 'goto-analytics-utc-instant' });
export const ReportScopeSchema = Type.Union([
Type.Object({ kind: Type.Literal('overview') }, exact),
Type.Object({ kind: Type.Literal('domain'), domain }, exact),
Type.Object({ kind: Type.Literal('link'), domain, path }, exact)
]);
export type ReportScope = Static<typeof ReportScopeSchema>;
export const VisitTotalsSchema = Type.Object({ validVisits: count, invalidVisits: count }, exact);
export type VisitTotals = Static<typeof VisitTotalsSchema>;
export const ReportPeriodSchema = Type.Object(
{
from: instant,
asOf: instant,
timezone: Type.Literal('UTC'),
retentionMonths: Type.Union([Type.Literal(3), Type.Literal(9), Type.Literal(12)])
},
exact
);
export type ReportPeriod = Static<typeof ReportPeriodSchema>;
const headers = {
version: Type.Literal(1),
period: ReportPeriodSchema,
totals: VisitTotalsSchema
};
export const OverviewSummarySchema = Type.Object(
{
...headers,
scope: Type.Object({ kind: Type.Literal('overview') }, exact),
base: VisitTotalsSchema,
domains: Type.Array(Type.Object({ domain, totals: VisitTotalsSchema }, exact))
},
exact
);
export type OverviewSummary = Static<typeof OverviewSummarySchema>;
export const DomainSummarySchema = Type.Object(
{
...headers,
scope: Type.Object({ kind: Type.Literal('domain'), domain }, exact),
base: VisitTotalsSchema,
links: Type.Array(Type.Object({ path, totals: VisitTotalsSchema }, exact))
},
exact
);
export type DomainSummary = Static<typeof DomainSummarySchema>;
export const LinkSummarySchema = Type.Object(
{
...headers,
scope: Type.Object({ kind: Type.Literal('link'), domain, path }, exact)
},
exact
);
export type LinkSummary = Static<typeof LinkSummarySchema>;
export const VisitSummarySchema = Type.Union([
OverviewSummarySchema,
DomainSummarySchema,
LinkSummarySchema
]);
export type VisitSummary = Static<typeof VisitSummarySchema>;
const familyCounts = Type.Object(
{
family: Type.String({ minLength: 1 }),
validVisits: count,
invalidVisits: count
},
exact
);
export const VisitDetailsSchema = Type.Object(
{
version: Type.Literal(1),
scope: ReportScopeSchema,
period: ReportPeriodSchema,
daily: Type.Array(
Type.Object(
{
start: instant,
end: instant,
partial: Type.Boolean(),
validVisits: count,
invalidVisits: count
},
exact
)
),
browsers: Type.Array(familyCounts),
operatingSystems: Type.Array(familyCounts)
},
exact
);
export type VisitDetails = Static<typeof VisitDetailsSchema>;
export interface PreparedVisitReport {
summary: VisitSummary;
details: VisitDetails;
}
export interface SerializedVisitReport {
scope: ReportScope;
summaryJson: string;
detailsJson: string;
}
export function zeroVisitTotals(): VisitTotals {
return { validVisits: 0, invalidVisits: 0 };
}
+73
View File
@@ -0,0 +1,73 @@
import { Value } from '@sinclair/typebox/value';
import { createVisitPeriod } from './period.js';
import {
VisitDetailsSchema,
VisitSummarySchema,
type ReportPeriod,
type ReportScope,
type VisitDetails,
type VisitSummary,
type VisitTotals
} from './reports.js';
export function checkSummary(value: unknown): VisitSummary {
if (!Value.Check(VisitSummarySchema, value)) throw invalidReport();
checkPeriod(value.period);
if ('base' in value && !withinTotals(value.base, value.totals)) throw invalidReport();
return value;
}
export function checkDetails(value: unknown): VisitDetails {
if (!Value.Check(VisitDetailsSchema, value)) throw invalidReport();
checkPeriod(value.period);
const from = Date.parse(value.period.from);
const asOf = Date.parse(value.period.asOf);
const dayMilliseconds = 86_400_000;
const earliestStart = Math.floor(from / dayMilliseconds) * dayMilliseconds;
const latestEnd = Math.ceil(asOf / dayMilliseconds) * dayMilliseconds;
for (const point of value.daily) {
const start = Date.parse(point.start);
const end = Date.parse(point.end);
if (start < earliestStart || end > latestEnd || start >= end || start >= asOf || end <= from) {
throw invalidReport();
}
}
return value;
}
function checkPeriod(period: ReportPeriod): void {
let expected: ReportPeriod;
try {
expected = createVisitPeriod(period.retentionMonths, Date.parse(period.asOf));
} catch {
throw invalidReport();
}
if (!samePeriod(period, expected)) throw invalidReport();
}
export function samePeriod(left: ReportPeriod, right: ReportPeriod): boolean {
return (
left.from === right.from &&
left.asOf === right.asOf &&
left.timezone === right.timezone &&
left.retentionMonths === right.retentionMonths
);
}
export function sameScope(left: ReportScope, right: ReportScope): boolean {
if (left.kind === 'overview') return right.kind === 'overview';
if (left.kind === 'domain') return right.kind === 'domain' && left.domain === right.domain;
return right.kind === 'link' && left.domain === right.domain && left.path === right.path;
}
export function equalTotals(left: VisitTotals, right: VisitTotals): boolean {
return left.validVisits === right.validVisits && left.invalidVisits === right.invalidVisits;
}
export function withinTotals(subset: VisitTotals, totals: VisitTotals): boolean {
return subset.validVisits <= totals.validVisits && subset.invalidVisits <= totals.invalidVisits;
}
export function invalidReport(): Error {
return new Error('Invalid analytics report');
}
+6
View File
@@ -0,0 +1,6 @@
{
"extends": "./tsconfig.json",
"compilerOptions": { "rootDir": "src" },
"include": ["src/**/*.ts"],
"exclude": ["dist", "src/**/*.test.ts"]
}
+11
View File
@@ -0,0 +1,11 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"declaration": true,
"rootDir": ".",
"outDir": "dist",
"types": ["node"]
},
"include": ["src/**/*.ts"],
"exclude": ["dist"]
}
+13
View File
@@ -0,0 +1,13 @@
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
include: ['src/**/*.test.ts'],
environment: 'node',
coverage: {
provider: 'v8',
include: ['src/**/*.ts'],
exclude: ['src/**/*.test.ts']
}
}
});
+1
View File
@@ -28,6 +28,7 @@
"test:coverage": "vitest run --coverage"
},
"dependencies": {
"@goto/analytics": "workspace:*",
"kysely": "catalog:"
},
"devDependencies": {
@@ -0,0 +1,41 @@
import { sql, type Kysely, type Migration } from 'kysely';
export const analyticsReports: Migration = {
async up(db: Kysely<unknown>): Promise<void> {
await db.schema
.createTable('analytics_reports')
.addColumn('id', 'integer', (column) => column.primaryKey())
.addColumn('scope_kind', 'text', (column) => column.notNull())
.addColumn('domain', 'text')
.addColumn('path', 'text')
.addColumn('summary_json', 'text', (column) => column.notNull())
.addColumn('details_json', 'text', (column) => column.notNull())
.addCheckConstraint(
'analytics_reports_scope',
sql`
(scope_kind = 'overview' AND domain IS NULL AND path IS NULL) OR
(scope_kind = 'domain' AND domain IS NOT NULL AND length(domain) > 0 AND path IS NULL) OR
(scope_kind = 'link' AND domain IS NOT NULL AND length(domain) > 0
AND path IS NOT NULL AND substr(path, 1, 1) = '/')
`
)
.addCheckConstraint('analytics_reports_summary_json', sql`json_valid(summary_json)`)
.addCheckConstraint('analytics_reports_details_json', sql`json_valid(details_json)`)
.execute();
await sql`CREATE UNIQUE INDEX analytics_reports_overview
ON analytics_reports (scope_kind) WHERE scope_kind = 'overview'`.execute(db);
await sql`CREATE UNIQUE INDEX analytics_reports_domain
ON analytics_reports (domain) WHERE scope_kind = 'domain'`.execute(db);
await sql`CREATE UNIQUE INDEX analytics_reports_link
ON analytics_reports (domain, path) WHERE scope_kind = 'link'`.execute(db);
await db.schema.dropTable('visit_reports').execute();
},
async down(db: Kysely<unknown>): Promise<void> {
await db.schema.dropTable('analytics_reports').execute();
await db.schema
.createTable('visit_reports')
.addColumn('key', 'text', (column) => column.primaryKey())
.addColumn('data', 'text', (column) => column.notNull())
.execute();
}
};
+3 -1
View File
@@ -4,6 +4,7 @@ import { visitRetention } from './0003_visit_retention.js';
import { removeVisitCollection } from './0004_remove_visit_collection.js';
import { addVisitOutcome } from './0005_visit_outcome.js';
import { visitReports } from './0006_visit_reports.js';
import { analyticsReports } from './0007_analytics_reports.js';
export const migrations = {
'0001_visits': initial,
@@ -11,5 +12,6 @@ export const migrations = {
'0003_visit_retention': visitRetention,
'0004_remove_visit_collection': removeVisitCollection,
'0005_visit_outcome': addVisitOutcome,
'0006_visit_reports': visitReports
'0006_visit_reports': visitReports,
'0007_analytics_reports': analyticsReports
};
+18 -10
View File
@@ -55,11 +55,17 @@ test('opens a real WAL database and applies versioned migrations once', async ()
'0003_visit_retention',
'0004_remove_visit_collection',
'0005_visit_outcome',
'0006_visit_reports'
'0006_visit_reports',
'0007_analytics_reports'
]);
const tables = await db.introspection.getTables();
const names = tables.map((table) => table.name);
expect(names).not.toContain('visit_collection');
expect(names).not.toContain('visit_reports');
expect(names).toContain('analytics_reports');
expect(
tables.find((table) => table.name === 'analytics_reports')?.columns.map(({ name }) => name)
).toEqual(['id', 'scope_kind', 'domain', 'path', 'summary_json', 'details_json']);
expect(await runMigrations(db)).toEqual([]);
} finally {
await db.destroy();
@@ -109,7 +115,7 @@ test('persists request-time outcomes for the same logical path on different doma
}
});
test('adds request-time labels without changing existing visit data', async () => {
test('upgrades legacy visits and discards only the obsolete report cache', async () => {
const db = await openDb(join(directory, 'legacy.sqlite'));
try {
const migrator = new Migrator({
@@ -127,13 +133,14 @@ test('adds request-time labels without changing existing visit data', async () =
VALUES (${visit.domain}, ${visit.path}, ${visit.visited_at}, ${visit.ip}, ${visit.user_agent}, ${visit.accept_language})
`.execute(db);
expect(await runMigrations(db)).toEqual([
'0002_visit_classification',
'0003_visit_retention',
'0004_remove_visit_collection',
'0005_visit_outcome',
'0006_visit_reports'
]);
const upgrade = await migrator.migrateTo('0006_visit_reports');
if (upgrade.error) throw upgrade.error;
await sql`INSERT INTO visit_reports (key, data) VALUES ('overview', 'obsolete cache')`.execute(
db
);
expect(await runMigrations(db)).toEqual(['0007_analytics_reports']);
expect(await db.selectFrom('analytics_reports').selectAll().execute()).toEqual([]);
expect(await db.selectFrom('visits').selectAll().executeTakeFirstOrThrow()).toEqual({
...visit,
id: 1,
@@ -171,7 +178,8 @@ test('a failed migration rolls back schema and bookkeeping so startup can retry'
'0003_visit_retention',
'0004_remove_visit_collection',
'0005_visit_outcome',
'0006_visit_reports'
'0006_visit_reports',
'0007_analytics_reports'
]);
expect(await runMigrations(db)).toEqual([]);
} finally {
+8 -4
View File
@@ -15,14 +15,18 @@ export interface VisitTable {
os: string;
}
export interface VisitReportTable {
key: string;
data: string;
export interface AnalyticsReportTable {
id: Generated<number>;
scope_kind: 'overview' | 'domain' | 'link';
domain: string | null;
path: string | null;
summary_json: string;
details_json: string;
}
export interface Database {
visits: VisitTable;
visit_reports: VisitReportTable;
analytics_reports: AnalyticsReportTable;
}
export type ClassifiedVisit = Omit<Insertable<VisitTable>, 'id'>;
+7 -10
View File
@@ -1,16 +1,13 @@
export { aggregateVisitReports } from './queries.js';
export { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js';
export { visitReportKey, zeroVisitCounts } from './types.js';
export {
clearVisitReports,
publishVisitReports,
readVisitSummary,
readVisitDetails
} from './storage.js';
export type {
AggregatedVisitReports,
DailyVisitCounts,
FamilyVisitCounts,
PreparedVisitReport,
ReportInventory,
ScopedFamilyCounts,
SerializedVisitReport,
StoredVisitReports,
TimeBucket,
VisitCounts,
VisitReportPeriod
ScopedFamilyCounts
} from './types.js';
+204 -51
View File
@@ -1,7 +1,7 @@
import { afterEach, beforeEach, expect, test } from 'vitest';
import { openDb, runMigrations, type ClassifiedVisit } from '../index.js';
import { aggregateVisitReports } from './queries.js';
import { visitReportKey, type ReportInventory } from './types.js';
import type { ReportInventory } from './types.js';
let db: Awaited<ReturnType<typeof openDb>>;
@@ -71,75 +71,228 @@ async function seedVisits(): Promise<void> {
.execute();
}
test('aggregates UTC days and stored families for overview, current domains and known paths', async () => {
test('aggregates explicit UTC scope identities, outcomes and root subsets', async () => {
await seedVisits();
const result = await aggregateVisitReports(db, since, asOf, inventory);
const overview = visitReportKey();
const domainScope = visitReportKey(domain);
const docs = visitReportKey(domain, '/docs');
const root = visitReportKey(domain, '/');
const otherDomainScope = visitReportKey(otherDomain);
const otherDocs = visitReportKey(otherDomain, '/docs');
expect(result.daily).toHaveLength(8);
expect(result.daily).toEqual(
expect.arrayContaining([
{ scope: overview, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
{ scope: overview, day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 },
{ scope: domainScope, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
{ scope: domainScope, day: '2026-02-01', total: 3, valid: 1, invalid: 2, base: 2 },
{ scope: docs, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
{ scope: root, day: '2026-02-01', total: 2, valid: 1, invalid: 1, base: 2 },
{ scope: otherDomainScope, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 },
{ scope: otherDocs, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 }
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-01-31',
validVisits: 1,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-02-01',
validVisits: 1,
invalidVisits: 4,
baseValidVisits: 1,
baseInvalidVisits: 2
},
{
scopeKind: 'domain',
domain,
path: null,
day: '2026-01-31',
validVisits: 1,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
scopeKind: 'domain',
domain,
path: null,
day: '2026-02-01',
validVisits: 1,
invalidVisits: 2,
baseValidVisits: 1,
baseInvalidVisits: 1
},
{
scopeKind: 'link',
domain,
path: '/docs',
day: '2026-01-31',
validVisits: 1,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
scopeKind: 'link',
domain,
path: '/',
day: '2026-02-01',
validVisits: 1,
invalidVisits: 1,
baseValidVisits: 1,
baseInvalidVisits: 1
},
{
scopeKind: 'domain',
domain: otherDomain,
path: null,
day: '2026-02-01',
validVisits: 0,
invalidVisits: 1,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
scopeKind: 'link',
domain: otherDomain,
path: '/docs',
day: '2026-02-01',
validVisits: 0,
invalidVisits: 1,
baseValidVisits: 0,
baseInvalidVisits: 0
}
])
);
expect(result.browsers.filter(({ scope }) => scope === overview)).toEqual([
{ scope: overview, name: 'Firefox', total: 4, valid: 1, invalid: 3, base: 2 },
{ scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
{ scope: overview, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
expect(result.browsers.filter(({ scopeKind }) => scopeKind === 'overview')).toEqual([
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Firefox',
validVisits: 1,
invalidVisits: 3
},
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Other',
validVisits: 0,
invalidVisits: 1
},
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Safari',
validVisits: 1,
invalidVisits: 0
}
]);
expect(result.operatingSystems.filter(({ scope }) => scope === overview)).toEqual([
{ scope: overview, name: 'Linux', total: 3, valid: 1, invalid: 2, base: 2 },
{ scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
{ scope: overview, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 },
{ scope: overview, name: 'macOS', total: 1, valid: 1, invalid: 0, base: 1 }
expect(result.operatingSystems.filter(({ scopeKind }) => scopeKind === 'overview')).toEqual([
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Linux',
validVisits: 1,
invalidVisits: 2
},
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Other',
validVisits: 0,
invalidVisits: 1
},
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'Windows',
validVisits: 0,
invalidVisits: 1
},
{
scopeKind: 'overview',
domain: null,
path: null,
family: 'macOS',
validVisits: 1,
invalidVisits: 0
}
]);
expect(result.browsers.filter(({ scope }) => scope === domainScope)).toEqual([
{ scope: domainScope, name: 'Firefox', total: 2, valid: 1, invalid: 1, base: 1 },
{ scope: domainScope, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
{ scope: domainScope, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
expect(
result.browsers.filter(({ scopeKind, path }) => scopeKind === 'link' && path === '/')
).toEqual([
{ scopeKind: 'link', domain, path: '/', family: 'Firefox', validVisits: 0, invalidVisits: 1 },
{ scopeKind: 'link', domain, path: '/', family: 'Safari', validVisits: 1, invalidVisits: 0 }
]);
expect(result.browsers.filter(({ scope }) => scope === root)).toEqual([
{ scope: root, name: 'Firefox', total: 1, valid: 0, invalid: 1, base: 1 },
{ scope: root, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
expect(
result.operatingSystems.filter(
({ scopeKind, domain: rowDomain }) => scopeKind === 'link' && rowDomain === otherDomain
)
).toEqual([
{
scopeKind: 'link',
domain: otherDomain,
path: '/docs',
family: 'Windows',
validVisits: 0,
invalidVisits: 1
}
]);
expect(result.operatingSystems.filter(({ scope }) => scope === otherDocs)).toEqual([
{ scope: otherDocs, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 }
]);
const familyScopes = new Set(
[...result.browsers, ...result.operatingSystems].map(({ scope }) => scope)
);
expect(familyScopes).toEqual(
new Set([overview, domainScope, docs, root, otherDomainScope, otherDocs])
);
});
test('empty inventory still aggregates historical visits only under overview', async () => {
test('empty inventory keeps historical overview traffic but no domain or link scopes', async () => {
await seedVisits();
const result = await aggregateVisitReports(db, since, asOf, []);
expect(result.daily).toEqual([
{ scope: 'overview', day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
{ scope: 'overview', day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 }
]);
expect(result.browsers.map(({ scope }) => scope)).toEqual(['overview', 'overview', 'overview']);
expect(result.operatingSystems.map(({ scope }) => scope)).toEqual([
'overview',
'overview',
'overview',
'overview'
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-01-31',
validVisits: 1,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
},
{
scopeKind: 'overview',
domain: null,
path: null,
day: '2026-02-01',
validVisits: 1,
invalidVisits: 4,
baseValidVisits: 1,
baseInvalidVisits: 2
}
]);
expect(
[...result.browsers, ...result.operatingSystems].every(
(row) => row.scopeKind === 'overview' && row.domain === null && row.path === null
)
).toBe(true);
});
test('paths remain exact bound identities, including punctuation and Unicode', async () => {
const path = '/café,"docs';
await db
.insertInto('visits')
.values({ ...visit, path })
.execute();
const result = await aggregateVisitReports(db, since, asOf, [{ domain, paths: [path] }]);
expect(result.daily.find((row) => row.scopeKind === 'link')).toEqual({
scopeKind: 'link',
domain,
path,
day: '2026-01-31',
validVisits: 1,
invalidVisits: 0,
baseValidVisits: 0,
baseInvalidVisits: 0
});
});
test('no visits produces no aggregate rows for configured scopes', async () => {
+27 -30
View File
@@ -1,11 +1,10 @@
import { sql, type Kysely } from 'kysely';
import type { Database } from '../types.js';
import {
visitReportKey,
type AggregatedVisitReports,
type DailyVisitCounts,
type ReportInventory,
type ScopedFamilyCounts
import type {
AggregatedVisitReports,
DailyVisitCounts,
ReportInventory,
ScopedFamilyCounts
} from './types.js';
export async function aggregateVisitReports(
@@ -15,31 +14,36 @@ export async function aggregateVisitReports(
inventory: ReportInventory
): Promise<AggregatedVisitReports> {
const scopes = scopedVisits(since, asOf, inventory);
const counts = visitCountColumns();
const identity = sql`scope_kind AS "scopeKind", domain, path`;
const counts = sql`
SUM(CASE WHEN outcome = 'valid' THEN 1 ELSE 0 END) AS "validVisits",
SUM(CASE WHEN outcome = 'invalid' THEN 1 ELSE 0 END) AS "invalidVisits"
`;
return db.transaction().execute(async (transaction) => {
const daily = await sql<DailyVisitCounts>`
${scopes}
SELECT scope, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts}
SELECT ${identity}, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts},
SUM(CASE WHEN visit_path = '/' AND outcome = 'valid' THEN 1 ELSE 0 END) AS "baseValidVisits",
SUM(CASE WHEN visit_path = '/' AND outcome = 'invalid' THEN 1 ELSE 0 END) AS "baseInvalidVisits"
FROM scoped_visits
GROUP BY scope, day
ORDER BY scope, day
GROUP BY scope_kind, domain, path, day
ORDER BY scope_kind, domain, path, day
`.execute(transaction);
const browsers = await sql<ScopedFamilyCounts>`
${scopes}
SELECT scope, browser AS name, ${counts}
SELECT ${identity}, browser AS family, ${counts}
FROM scoped_visits
GROUP BY scope, browser
ORDER BY scope, name
GROUP BY scope_kind, domain, path, browser
ORDER BY scope_kind, domain, path, family
`.execute(transaction);
const operatingSystems = await sql<ScopedFamilyCounts>`
${scopes}
SELECT scope, os AS name, ${counts}
SELECT ${identity}, os AS family, ${counts}
FROM scoped_visits
GROUP BY scope, os
ORDER BY scope, name
GROUP BY scope_kind, domain, path, os
ORDER BY scope_kind, domain, path, family
`.execute(transaction);
return {
daily: daily.rows,
browsers: browsers.rows,
@@ -56,32 +60,25 @@ function scopedVisits(since: number, asOf: number, inventory: ReportInventory) {
return sql`(domain = ${domain} AND path IN (${sql.join(currentPaths)}))`;
});
const currentLink = linkRules.length === 0 ? sql`0` : sql.join(linkRules, sql` OR `);
return sql`
WITH retained_visits AS (
SELECT domain, path, outcome, visited_at, browser, os
FROM visits
WHERE visited_at >= ${since} AND visited_at < ${asOf}
), scoped_visits AS (
SELECT ${visitReportKey()} AS scope, path, outcome, visited_at, browser, os
SELECT 'overview' AS scope_kind, NULL AS domain, NULL AS path,
path AS visit_path, outcome, visited_at, browser, os
FROM retained_visits
UNION ALL
SELECT json_array(domain) AS scope, path, outcome, visited_at, browser, os
SELECT 'domain' AS scope_kind, domain, NULL AS path,
path AS visit_path, outcome, visited_at, browser, os
FROM retained_visits
WHERE ${currentDomain}
UNION ALL
SELECT json_array(domain, path) AS scope, path, outcome, visited_at, browser, os
SELECT 'link' AS scope_kind, domain, path,
path AS visit_path, outcome, visited_at, browser, os
FROM retained_visits
WHERE ${currentLink}
)
`;
}
function visitCountColumns() {
return sql`
COUNT(*) AS total,
SUM(CASE WHEN outcome = ${'valid'} THEN 1 ELSE 0 END) AS valid,
SUM(CASE WHEN outcome = ${'invalid'} THEN 1 ELSE 0 END) AS invalid,
SUM(CASE WHEN path = ${'/'} THEN 1 ELSE 0 END) AS base
`;
}
+194 -128
View File
@@ -1,27 +1,30 @@
import { afterEach, beforeEach, expect, test } from 'vitest';
import { openDb, runMigrations } from '../index.js';
import { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js';
import {
visitReportKey,
zeroVisitCounts,
createVisitPeriod,
encodeVisitReports,
visitDays,
zeroVisitTotals,
type PreparedVisitReport,
type SerializedVisitReport,
type VisitCounts,
type VisitReportPeriod
} from './types.js';
type ReportScope,
type VisitSummary,
type VisitTotals
} from '@goto/analytics';
import { DatabaseSync } from 'node:sqlite';
import { sql } from 'kysely';
import { afterEach, beforeEach, expect, test, vi } from 'vitest';
import { openDb, runMigrations } from '../index.js';
import {
clearVisitReports,
publishVisitReports,
readVisitDetails,
readVisitSummary
} from './storage.js';
let db: Awaited<ReturnType<typeof openDb>>;
const domain = 'go.example.test';
const period: VisitReportPeriod = {
timezone: 'UTC',
retentionMonths: 3,
since: Date.UTC(2026, 0, 1),
asOf: Date.UTC(2026, 0, 2),
days: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 0, 2), partial: false }],
weeks: [{ start: Date.UTC(2025, 11, 29), end: Date.UTC(2026, 0, 5), partial: true }],
months: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 1, 1), partial: true }]
};
const overviewScope = { kind: 'overview' } as const;
const domainScope = { kind: 'domain', domain } as const;
const rootScope = { kind: 'link', domain, path: '/' } as const;
const asOf = Date.UTC(2026, 3, 1);
beforeEach(async () => {
db = await openDb(':memory:');
@@ -32,132 +35,195 @@ afterEach(async () => {
await db.destroy();
});
function preparedReport(counts: VisitCounts = zeroVisitCounts()): PreparedVisitReport {
function preparedReport(
scope: ReportScope,
cutoff = asOf,
totals: VisitTotals = zeroVisitTotals()
): PreparedVisitReport {
const period = createVisitPeriod(3, cutoff);
let summary: VisitSummary;
if (scope.kind === 'overview') {
summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), domains: [] };
} else if (scope.kind === 'domain') {
summary = { version: 1, scope, period, totals, base: zeroVisitTotals(), links: [] };
} else {
summary = { version: 1, scope, period, totals };
}
const populated = totals.validVisits + totals.invalidVisits > 0;
return {
totals: counts,
daily: [counts],
weekly: [counts],
monthly: [counts],
browsers: [],
operatingSystems: [],
domainCounts: [],
linkCounts: []
summary,
details: {
version: 1,
scope,
period,
daily: visitDays(period).map((day, index) => ({
...day,
...(index === 0 ? totals : zeroVisitTotals())
})),
browsers: populated ? [{ family: 'Firefox', ...totals }] : [],
operatingSystems: populated ? [{ family: 'Linux', ...totals }] : []
}
};
}
function serializedReport(key: string, report: PreparedVisitReport): SerializedVisitReport {
return { key, data: JSON.stringify(report) };
}
test('reads summary and details for each explicit SQL scope including root', async () => {
const reports = [
preparedReport(overviewScope, asOf, { validVisits: 2, invalidVisits: 1 }),
preparedReport(domainScope, asOf, { validVisits: 1, invalidVisits: 1 }),
preparedReport(rootScope)
];
await publishVisitReports(db, encodeVisitReports(reports));
test('reads the shared period, scope report and selected path from prepared rows', async () => {
const selection = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
selection.browsers = [{ name: 'Firefox', counts: selection.totals }];
selection.operatingSystems = [{ name: 'Linux', counts: selection.totals }];
const domainReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 1 });
domainReport.linkCounts = [{ path: '/docs', counts: selection.totals }];
const overview = preparedReport({ total: 3, valid: 2, invalid: 1, base: 1 });
overview.domainCounts = [{ domain, counts: domainReport.totals }];
await publishVisitReports(db, period, [
serializedReport('empty', preparedReport()),
serializedReport(visitReportKey(), overview),
serializedReport(visitReportKey(domain), domainReport),
serializedReport(visitReportKey(domain, '/docs'), selection)
for (const report of reports) {
expect(await readVisitSummary(db, report.summary.scope)).toEqual(report.summary);
expect(await readVisitDetails(db, report.summary.scope)).toEqual(report.details);
}
expect(
await db
.selectFrom('analytics_reports')
.select(['scope_kind', 'domain', 'path'])
.orderBy('id')
.execute()
).toEqual([
{ scope_kind: 'overview', domain: null, path: null },
{ scope_kind: 'domain', domain, path: null },
{ scope_kind: 'link', domain, path: '/' }
]);
expect(await readPreparedVisitReports(db)).toEqual({
period,
report: overview,
selection: overview
});
expect(await readPreparedVisitReports(db, domain)).toEqual({
period,
report: domainReport,
selection: domainReport
});
expect(await readPreparedVisitReports(db, domain, '/docs')).toEqual({
period,
report: domainReport,
selection
});
});
test('publication replaces all output rows and a failed replacement preserves the previous set', async () => {
const oldReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
await publishVisitReports(db, period, [
serializedReport(visitReportKey(), oldReport),
serializedReport(visitReportKey(domain), oldReport)
]);
const replacementPeriod = { ...period, retentionMonths: 9 as const };
const replacementReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 0 });
const replacementRows = [serializedReport(visitReportKey(), replacementReport)];
await publishVisitReports(db, replacementPeriod, replacementRows);
test('publication replaces the complete batch, including when the replacement is empty', async () => {
await publishVisitReports(
db,
encodeVisitReports([preparedReport(overviewScope), preparedReport(domainScope)])
);
const replacement = preparedReport(overviewScope, asOf + 86_400_000);
await publishVisitReports(db, encodeVisitReports([replacement]));
expect(await db.selectFrom('visit_reports').select('key').orderBy('key').execute()).toEqual([
{ key: 'overview' },
{ key: 'period' }
]);
expect(await readPreparedVisitReports(db, domain)).toBeNull();
await expect(
publishVisitReports(db, period, [
serializedReport(visitReportKey(), oldReport),
serializedReport(visitReportKey(), oldReport)
])
).rejects.toThrow();
expect(await readPreparedVisitReports(db)).toEqual({
period: replacementPeriod,
report: replacementReport,
selection: replacementReport
});
expect(await readVisitSummary(db, overviewScope)).toEqual(replacement.summary);
expect(await readVisitSummary(db, domainScope)).toBeNull();
await publishVisitReports(db, []);
expect(await db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
});
test('a published zero-count path uses the empty template but a missing domain stays unavailable', async () => {
const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 1 });
domainReport.linkCounts = [{ path: '/new', counts: zeroVisitCounts() }];
const empty = preparedReport();
await publishVisitReports(db, period, [
serializedReport(visitReportKey(domain), domainReport),
serializedReport('empty', empty)
]);
test('native uniqueness and JSON failures roll back publication deletion', async () => {
const original = preparedReport(overviewScope);
await publishVisitReports(db, encodeVisitReports([original]));
const [replacement] = encodeVisitReports([preparedReport(domainScope)]);
if (!replacement) throw new Error('Expected an encoded report fixture');
expect(await readPreparedVisitReports(db, domain, '/new')).toEqual({
period,
report: domainReport,
selection: empty
});
expect(await readPreparedVisitReports(db, 'new.example.test', '/new')).toBeNull();
for (const batch of [
[replacement, replacement],
[{ ...replacement, summaryJson: '{' }],
[{ ...replacement, detailsJson: '{' }]
]) {
await expect(publishVisitReports(db, batch)).rejects.toThrow();
expect(await readVisitSummary(db, overviewScope)).toEqual(original.summary);
expect(await readVisitDetails(db, overviewScope)).toEqual(original.details);
expect(await readVisitSummary(db, domainScope)).toBeNull();
}
});
test('missing link bodies stay unavailable for unpublished paths and populated links', async () => {
const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
domainReport.linkCounts = [{ path: '/docs', counts: domainReport.totals }];
await publishVisitReports(db, period, [
serializedReport(visitReportKey(domain), domainReport),
serializedReport('empty', preparedReport())
]);
test('missing scopes are null, not implicit empty reports or parent fallbacks', async () => {
const report = preparedReport(domainScope);
if (!('links' in report.summary)) throw new Error('Expected a domain fixture');
report.summary.links = [{ path: '/new', totals: zeroVisitTotals() }];
await publishVisitReports(db, encodeVisitReports([report]));
expect(await readPreparedVisitReports(db, domain, '/new')).toBeNull();
expect(await readPreparedVisitReports(db, domain, '/docs')).toBeNull();
});
test('missing metadata is unavailable and clearing deletes every output row', async () => {
expect(await readPreparedVisitReports(db)).toBeNull();
await db
.insertInto('visit_reports')
.values(serializedReport(visitReportKey(), preparedReport()))
.execute();
expect(await readPreparedVisitReports(db)).toBeNull();
await publishVisitReports(db, period, [
serializedReport(visitReportKey(), preparedReport()),
serializedReport('empty', preparedReport())
]);
expect(await readVisitSummary(db, overviewScope)).toBeNull();
expect(await readVisitSummary(db, { kind: 'link', domain, path: '/new' })).toBeNull();
expect(await readVisitDetails(db, rootScope)).toBeNull();
await clearVisitReports(db);
expect(await db.selectFrom('visit_reports').selectAll().execute()).toEqual([]);
expect(await readPreparedVisitReports(db)).toBeNull();
expect(await readVisitSummary(db, domainScope)).toBeNull();
expect(await db.selectFrom('analytics_reports').select('id').execute()).toEqual([]);
});
test('malformed stored JSON is left for the caller to handle', async () => {
await publishVisitReports(db, period, [{ key: visitReportKey(), data: '{' }]);
test('summary and details readers select and decode only their own column', async () => {
const report = preparedReport(overviewScope);
const rows = encodeVisitReports([report]);
const [row] = rows;
if (!row) throw new Error('Expected an encoded report fixture');
await publishVisitReports(db, rows);
await db.updateTable('analytics_reports').set({ details_json: '{}' }).execute();
await expect(readPreparedVisitReports(db)).rejects.toThrow(SyntaxError);
const statements = vi.spyOn(DatabaseSync.prototype, 'prepare');
try {
expect(await readVisitSummary(db, overviewScope)).toEqual(report.summary);
expect(statements.mock.calls.map(([query]) => query)).toEqual([
expect.stringMatching(/^select\s+"summary_json"\s+from\s+/i)
]);
statements.mockClear();
await expect(readVisitDetails(db, overviewScope)).rejects.toThrow('Invalid analytics report');
expect(statements.mock.calls.map(([query]) => query)).toEqual([
expect.stringMatching(/^select\s+"details_json"\s+from\s+/i)
]);
} finally {
statements.mockRestore();
}
await db
.updateTable('analytics_reports')
.set({ summary_json: '{}', details_json: row.detailsJson })
.execute();
expect(await readVisitDetails(db, overviewScope)).toEqual(report.details);
await expect(readVisitSummary(db, overviewScope)).rejects.toThrow('Invalid analytics report');
});
test('database constraints reject invalid scope shapes, duplicate identities and malformed JSON', async () => {
const reports = encodeVisitReports(
[overviewScope, domainScope, rootScope].map((scope) => preparedReport(scope))
);
await publishVisitReports(db, reports);
for (const report of reports) {
const scope = report.scope;
await expect(
db
.insertInto('analytics_reports')
.values({
scope_kind: scope.kind,
domain: scope.kind === 'overview' ? null : scope.domain,
path: scope.kind === 'link' ? scope.path : null,
summary_json: report.summaryJson,
details_json: report.detailsJson
})
.execute()
).rejects.toThrow();
}
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('overview', 'bad.example.test', NULL, '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('domain', NULL, NULL, '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('link', 'go.example.test', NULL, '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('unknown', NULL, NULL, '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('domain', '', NULL, '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('link', '', '/docs', '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('link', 'go.example.test', '', '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
sql`INSERT INTO analytics_reports (scope_kind, domain, path, summary_json, details_json)
VALUES ('link', 'go.example.test', 'docs', '{}', '{}')`.execute(db)
).rejects.toThrow();
await expect(
db.updateTable('analytics_reports').set({ summary_json: '{' }).execute()
).rejects.toThrow();
await expect(
db.updateTable('analytics_reports').set({ details_json: '{' }).execute()
).rejects.toThrow();
});
+60 -41
View File
@@ -1,59 +1,78 @@
import {
parseVisitDetails,
parseVisitSummary,
type ReportScope,
type SerializedVisitReport,
type VisitDetails,
type VisitSummary
} from '@goto/analytics';
import type { Kysely } from 'kysely';
import type { Database } from '../types.js';
import {
visitReportKey,
type PreparedVisitReport,
type SerializedVisitReport,
type StoredVisitReports,
type VisitReportPeriod
} from './types.js';
export async function clearVisitReports(db: Kysely<Database>): Promise<void> {
await db.deleteFrom('visit_reports').execute();
await db.deleteFrom('analytics_reports').execute();
}
export async function publishVisitReports(
db: Kysely<Database>,
period: VisitReportPeriod,
reports: SerializedVisitReport[]
): Promise<void> {
const rows = [{ key: 'period', data: JSON.stringify(period) }, ...reports];
const rows = reports.map(({ scope, summaryJson, detailsJson }) => {
let domain: string | null = null;
let path: string | null = null;
if (scope.kind !== 'overview') domain = scope.domain;
if (scope.kind === 'link') path = scope.path;
return {
scope_kind: scope.kind,
domain,
path,
summary_json: summaryJson,
details_json: detailsJson
};
});
await db.transaction().execute(async (transaction) => {
await transaction.deleteFrom('visit_reports').execute();
await transaction.insertInto('visit_reports').values(rows).execute();
await transaction.deleteFrom('analytics_reports').execute();
if (rows.length > 0) await transaction.insertInto('analytics_reports').values(rows).execute();
});
}
export async function readPreparedVisitReports(
export async function readVisitSummary(
db: Kysely<Database>,
domain?: string,
path?: string
): Promise<StoredVisitReports | null> {
const reportKey = visitReportKey(domain);
const selectionKey = visitReportKey(domain, path);
const keys = [...new Set(['period', reportKey, selectionKey, 'empty'])];
const rows = await db.selectFrom('visit_reports').selectAll().where('key', 'in', keys).execute();
const dataByKey = new Map(rows.map(({ key, data }) => [key, data]));
const periodData = dataByKey.get('period');
const reportData = dataByKey.get(reportKey);
if (periodData === undefined || reportData === undefined) return null;
scope: ReportScope
): Promise<VisitSummary | null> {
const row = await reportScopeQuery(db, scope).select('summary_json').executeTakeFirst();
return row ? parseVisitSummary(row.summary_json, scope) : null;
}
const report = JSON.parse(reportData) as PreparedVisitReport;
let selection = report;
if (selectionKey !== reportKey) {
let selectionData = dataByKey.get(selectionKey);
if (selectionData === undefined) {
const linkCounts = report.linkCounts.find((link) => link.path === path)?.counts;
if (linkCounts?.total !== 0) return null;
selectionData = dataByKey.get('empty');
}
if (selectionData === undefined) return null;
selection = JSON.parse(selectionData) as PreparedVisitReport;
export async function readVisitDetails(
db: Kysely<Database>,
scope: ReportScope
): Promise<VisitDetails | null> {
const row = await reportScopeQuery(db, scope).select('details_json').executeTakeFirst();
return row ? parseVisitDetails(row.details_json, scope) : null;
}
function reportScopeQuery(db: Kysely<Database>, scope: ReportScope) {
if (scope.kind === 'overview') {
return db
.selectFrom('analytics_reports')
.where('scope_kind', '=', 'overview')
.where('domain', 'is', null)
.where('path', 'is', null);
}
if (scope.kind !== 'domain' && scope.kind !== 'link') {
throw new Error('Invalid analytics report');
}
if (typeof scope.domain !== 'string' || scope.domain.length === 0) {
throw new Error('Invalid analytics report');
}
const query = db
.selectFrom('analytics_reports')
.where('scope_kind', '=', scope.kind)
.where('domain', '=', scope.domain);
if (scope.kind === 'domain') return query.where('path', 'is', null);
if (typeof scope.path !== 'string' || !scope.path.startsWith('/')) {
throw new Error('Invalid analytics report');
}
return {
period: JSON.parse(periodData) as VisitReportPeriod,
report,
selection
};
return query.where('path', '=', scope.path);
}
@@ -1,20 +0,0 @@
import { expect, test } from 'vitest';
import { visitReportKey, zeroVisitCounts } from './types.js';
test('zero counts are fresh objects', () => {
const first = zeroVisitCounts();
const second = zeroVisitCounts();
first.total = 1;
expect(second).toEqual({ total: 0, valid: 0, invalid: 0, base: 0 });
expect(first).not.toBe(second);
});
test('scope keys use overview or JSON-encoded domains and exact paths', () => {
expect(visitReportKey()).toBe('overview');
expect(visitReportKey('go.example.test')).toBe('["go.example.test"]');
expect(visitReportKey('go.example.test', '/')).toBe('["go.example.test","/"]');
expect(visitReportKey('go.example.test', '/café,"docs')).toBe(
JSON.stringify(['go.example.test', '/café,"docs'])
);
});
+12 -65
View File
@@ -1,73 +1,20 @@
export interface VisitCounts {
total: number;
valid: number;
invalid: number;
base: number;
}
export function zeroVisitCounts(): VisitCounts {
return { total: 0, valid: 0, invalid: 0, base: 0 };
}
export interface TimeBucket {
start: number;
end: number;
partial: boolean;
}
export interface VisitReportPeriod {
timezone: 'UTC';
retentionMonths: 3 | 9 | 12;
since: number;
asOf: number;
days: TimeBucket[];
weeks: TimeBucket[];
months: TimeBucket[];
}
export interface FamilyVisitCounts {
name: string;
counts: VisitCounts;
}
export interface PreparedVisitReport {
totals: VisitCounts;
daily: VisitCounts[];
weekly: VisitCounts[];
monthly: VisitCounts[];
browsers: FamilyVisitCounts[];
operatingSystems: FamilyVisitCounts[];
domainCounts: Array<{ domain: string; counts: VisitCounts }>;
linkCounts: Array<{ path: string; counts: VisitCounts }>;
}
import type { VisitTotals } from '@goto/analytics';
export type ReportInventory = Array<{ domain: string; paths: string[] }>;
export function visitReportKey(domain?: string, path?: string): string {
if (domain === undefined) return 'overview';
return JSON.stringify(path === undefined ? [domain] : [domain, path]);
}
type DiscriminatedAggregateScope =
| { scopeKind: 'overview'; domain: null; path: null }
| { scopeKind: 'domain'; domain: string; path: null }
| { scopeKind: 'link'; domain: string; path: string };
export interface SerializedVisitReport {
key: string;
data: string;
}
export type DailyVisitCounts = DiscriminatedAggregateScope &
VisitTotals & {
day: string;
baseValidVisits: number;
baseInvalidVisits: number;
};
export interface StoredVisitReports {
period: VisitReportPeriod;
report: PreparedVisitReport;
selection: PreparedVisitReport;
}
export interface DailyVisitCounts extends VisitCounts {
scope: string;
day: string;
}
export interface ScopedFamilyCounts extends VisitCounts {
scope: string;
name: string;
}
export type ScopedFamilyCounts = DiscriminatedAggregateScope & VisitTotals & { family: string };
export interface AggregatedVisitReports {
daily: DailyVisitCounts[];
+31
View File
@@ -166,6 +166,9 @@ importers:
apps/web:
dependencies:
'@goto/analytics':
specifier: workspace:*
version: link:../../packages/analytics
'@goto/config':
specifier: workspace:*
version: link:../../packages/config
@@ -215,6 +218,9 @@ importers:
apps/worker:
dependencies:
'@goto/analytics':
specifier: workspace:*
version: link:../../packages/analytics
'@goto/config':
specifier: workspace:*
version: link:../../packages/config
@@ -244,6 +250,28 @@ importers:
specifier: 'catalog:'
version: 4.1.7(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.2)(yaml@2.9.1))
packages/analytics:
dependencies:
'@sinclair/typebox':
specifier: 'catalog:'
version: 0.34.49
devDependencies:
'@types/node':
specifier: 'catalog:'
version: 24.12.4
'@vitest/coverage-v8':
specifier: 'catalog:'
version: 4.1.7(vitest@4.1.7)
prettier:
specifier: 'catalog:'
version: 3.8.3
typescript:
specifier: 'catalog:'
version: 6.0.3
vitest:
specifier: 'catalog:'
version: 4.1.7(@types/node@24.12.4)(@vitest/coverage-v8@4.1.7)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.2)(yaml@2.9.1))
packages/config:
dependencies:
'@sinclair/typebox':
@@ -271,6 +299,9 @@ importers:
packages/db:
dependencies:
'@goto/analytics':
specifier: workspace:*
version: link:../analytics
kysely:
specifier: 'catalog:'
version: 0.28.17