feat(admin): add prepared analytics dashboards #14
No files matched your search
@@ -0,0 +1,7 @@
|
||||
---
|
||||
'@goto/web': minor
|
||||
'@goto/worker': minor
|
||||
'@goto/cli': minor
|
||||
---
|
||||
|
||||
Add an authenticated admin overview, domain summaries and dedicated link pages with current inventory and retained visits. One hourly background job, with scheduled jitter, prepares all reports in SQLite and coordinates retention cleanup. Admin pages read prepared snapshots directly and display their update time; no reporting API, token or worker URL is required. The job also prepares UTC daily, weekly and monthly series and browser/OS family counts for later visualization. Charts and the Health page are not included.
|
||||
@@ -9,6 +9,9 @@ on:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- packages/db/**
|
||||
- packages/config/**
|
||||
- scripts/**
|
||||
- tools/**
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
@@ -17,6 +20,9 @@ on:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- packages/db/**
|
||||
- packages/config/**
|
||||
- scripts/**
|
||||
- tools/**
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -48,3 +54,8 @@ jobs:
|
||||
run: pnpm --filter @goto/db lint
|
||||
- name: Unit tests
|
||||
run: pnpm --filter @goto/db test:coverage
|
||||
- name: Development script checks
|
||||
run: |
|
||||
pnpm exec prettier --check scripts
|
||||
pnpm exec eslint scripts
|
||||
pnpm test:scripts
|
||||
@@ -11,6 +11,7 @@ on:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- apps/web/**
|
||||
- apps/worker/**
|
||||
- packages/**
|
||||
pull_request:
|
||||
branches: [main]
|
||||
@@ -21,6 +22,7 @@ on:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- apps/web/**
|
||||
- apps/worker/**
|
||||
- packages/**
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -12,6 +12,9 @@ when:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- packages/db/**
|
||||
- packages/config/**
|
||||
- scripts/**
|
||||
- tools/**
|
||||
- event: pull_request
|
||||
branch: main
|
||||
path:
|
||||
@@ -21,6 +24,9 @@ when:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- packages/db/**
|
||||
- packages/config/**
|
||||
- scripts/**
|
||||
- tools/**
|
||||
|
||||
steps:
|
||||
prepare:
|
||||
@@ -52,6 +58,8 @@ steps:
|
||||
- corepack enable
|
||||
- pnpm --filter @goto/db check
|
||||
- pnpm --filter @goto/db lint
|
||||
- pnpm exec prettier --check scripts
|
||||
- pnpm exec eslint scripts
|
||||
|
||||
test:
|
||||
image: node:26.5.0-bookworm
|
||||
@@ -61,3 +69,4 @@ steps:
|
||||
- npm install --global corepack@0.36.0
|
||||
- corepack enable
|
||||
- pnpm --filter @goto/db test:coverage
|
||||
- pnpm test:scripts
|
||||
@@ -13,6 +13,7 @@ when:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- apps/web/**
|
||||
- apps/worker/**
|
||||
- packages/**
|
||||
- event: pull_request
|
||||
path:
|
||||
@@ -23,6 +24,7 @@ when:
|
||||
- .github/workflows/**
|
||||
- .woodpecker/**
|
||||
- apps/web/**
|
||||
- apps/worker/**
|
||||
- packages/**
|
||||
- event: tag
|
||||
ref: 'refs/tags/@goto/web@*'
|
||||
|
||||
+4
-1
@@ -20,10 +20,13 @@ COPY pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm fetch --frozen-lockfile
|
||||
COPY . .
|
||||
RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm install --offline --frozen-lockfile
|
||||
# Only generated output and caches are writable; source binds stay read-only.
|
||||
RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/db/dist /m/apps/web/.svelte-kit \
|
||||
'/m/apps/web/node_modules/$app' \
|
||||
/m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp \
|
||||
&& chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/db/dist \
|
||||
/m/apps/web/.svelte-kit /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp
|
||||
/m/apps/web/.svelte-kit '/m/apps/web/node_modules/$app' \
|
||||
/m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp
|
||||
USER ${LOCAL_UID}:${LOCAL_GID}
|
||||
WORKDIR /m/apps/web
|
||||
EXPOSE 5173
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { expect, test } from './support/test.ts';
|
||||
|
||||
test('shared credentials protect the read-only inventory and sign-out removes access', async ({
|
||||
test.use({ withWorker: true });
|
||||
|
||||
test('shared credentials protect overview, domain and link pages and sign-out removes access', async ({
|
||||
page,
|
||||
appServer
|
||||
}) => {
|
||||
@@ -13,16 +15,82 @@ test('shared credentials protect the read-only inventory and sign-out removes ac
|
||||
|
||||
await page.getByLabel('Password', { exact: true }).fill('e2e-only-password');
|
||||
await page.getByRole('button', { name: 'Sign in', exact: true }).click();
|
||||
await expect(page.getByRole('heading', { name: 'Configured links', exact: true })).toBeVisible();
|
||||
await expect(page.getByRole('heading', { name: 'Overview', exact: true })).toBeVisible();
|
||||
const visitsValue = page
|
||||
.getByRole('region', { name: 'Valid visits', exact: true })
|
||||
.locator('.admin-stat-value');
|
||||
await expect(page.getByRole('region', { name: 'Domains', exact: true })).toContainText('2');
|
||||
await expect(visitsValue).toHaveText('3');
|
||||
const retentionValue = page
|
||||
.getByRole('region', { name: 'Retention', exact: true })
|
||||
.locator('.admin-stat-value');
|
||||
await expect(retentionValue).toHaveText('3 months');
|
||||
await expect(page.locator('.admin-cards').getByRole('region')).toHaveCount(4);
|
||||
const domainLink = page.getByRole('link', { name: 'go.example.org', exact: true });
|
||||
const domainRow = page.getByRole('row').filter({ has: domainLink });
|
||||
await expect(domainRow.getByRole('cell')).toHaveText(['6', '2']);
|
||||
await domainLink.click();
|
||||
await expect(page.getByRole('heading', { name: 'go.example.org', exact: true })).toBeVisible();
|
||||
await expect(visitsValue).toHaveText('2');
|
||||
await expect(page.locator('.admin-cards .admin-stat-label')).toHaveText([
|
||||
'Links',
|
||||
'Valid visits',
|
||||
'Invalid clicks',
|
||||
'Retention'
|
||||
]);
|
||||
await expect(
|
||||
page
|
||||
.getByRole('region', { name: 'Links on go.example.org', exact: true })
|
||||
.getByRole('link', { name: 'https://example.org/first-domain', exact: true })
|
||||
page.getByRole('region', { name: 'Invalid clicks', exact: true }).locator('.admin-stat-value')
|
||||
).toHaveText('1');
|
||||
const linksTable = page.getByRole('region', { name: 'Links on go.example.org', exact: true });
|
||||
const baseTable = page.getByRole('region', { name: 'Base URL on go.example.org', exact: true });
|
||||
const columns = ['Path', 'Destination', 'State', 'Visits'];
|
||||
await expect(linksTable.getByRole('columnheader')).toHaveText(columns);
|
||||
const historicalInvalid = linksTable
|
||||
.getByRole('row')
|
||||
.filter({ has: page.getByRole('link', { name: '/a/b', exact: true }) });
|
||||
await expect(historicalInvalid.getByRole('cell').last()).toHaveText('0');
|
||||
await expect(baseTable.getByRole('columnheader')).toHaveText(columns);
|
||||
await expect(baseTable.getByRole('cell')).toHaveText(['404 Not found', 'Fallback', '0']);
|
||||
await expect(
|
||||
linksTable.getByRole('link', { name: 'https://example.org/first-domain', exact: true })
|
||||
).toBeVisible();
|
||||
|
||||
const domainBreadcrumb = page
|
||||
.getByRole('navigation', { name: 'Breadcrumb', exact: true })
|
||||
.getByRole('link', { name: 'go.example.org', exact: true });
|
||||
const docsPath = '/admin/domains/go.example.org/links/docs';
|
||||
await page.getByRole('link', { name: '/docs', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${appServer.origin}${docsPath}`);
|
||||
await expect(page.getByRole('heading', { name: '/docs', exact: true })).toBeVisible();
|
||||
await expect(visitsValue).toHaveText('1');
|
||||
await expect(
|
||||
page.getByRole('region', { name: 'Links on go.example.org', exact: true })
|
||||
).toHaveCount(0);
|
||||
|
||||
await domainBreadcrumb.click();
|
||||
await page.getByRole('link', { name: '/a%2Fb', exact: true }).click();
|
||||
await expect(page.getByRole('heading', { name: '/a%2Fb', exact: true })).toBeVisible();
|
||||
await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links/a%252Fb`);
|
||||
await expect(visitsValue).toHaveText('1');
|
||||
await domainBreadcrumb.click();
|
||||
await page.getByRole('link', { name: '/a/b', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links/a/b`);
|
||||
await expect(page.getByRole('heading', { name: '/a/b', exact: true })).toBeVisible();
|
||||
await expect(visitsValue).toHaveText('0');
|
||||
|
||||
await domainBreadcrumb.click();
|
||||
await baseTable.getByRole('link', { name: '/', exact: true }).click();
|
||||
await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links`);
|
||||
await expect(page.getByRole('heading', { name: '/', exact: true })).toBeVisible();
|
||||
await expect(visitsValue).toHaveText('0');
|
||||
await domainBreadcrumb.click();
|
||||
await expect(visitsValue).toHaveText('2');
|
||||
|
||||
const html = await page.content();
|
||||
await expect(html).not.toContain('e2e-only-password');
|
||||
await page.getByRole('button', { name: 'Sign out', exact: true }).click();
|
||||
await expect(page).toHaveURL(/\/signin\?/);
|
||||
await page.goto(`${appServer.origin}/admin`);
|
||||
await page.goto(`${appServer.origin}${docsPath}`);
|
||||
await expect(page).toHaveURL(/\/signin\?/);
|
||||
await expect(page.getByLabel('Username', { exact: true })).toBeVisible();
|
||||
});
|
||||
+83
-23
@@ -1,11 +1,12 @@
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { fork } from 'node:child_process';
|
||||
import { insertVisit, openDb, runMigrations } from '@goto/db';
|
||||
import { createServer } from 'node:net';
|
||||
import { join } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
const directory = process.env.GOTO_E2E_BUILD_DIR;
|
||||
if (!directory) throw new Error('Missing E2E build directory');
|
||||
|
||||
const runtimeDirectory = join(directory, `runtime-${process.pid}`);
|
||||
await mkdir(join(runtimeDirectory, 'config'), { recursive: true });
|
||||
process.chdir(runtimeDirectory);
|
||||
@@ -22,27 +23,86 @@ await new Promise((resolve, reject) =>
|
||||
reservation.close((error) => (error ? reject(error) : resolve()))
|
||||
);
|
||||
process.env.PORT = String(address.port);
|
||||
await writeFile(
|
||||
join(runtimeDirectory, 'config/goto.yaml'),
|
||||
JSON.stringify({
|
||||
site: { url: `http://localhost:${address.port}` },
|
||||
auth: { credentials: { username: 'example', password: 'e2e-only-password' } },
|
||||
domains: {
|
||||
'go.example.org': {
|
||||
links: {
|
||||
'/schema.json': { url: 'https://example.org/public-schema' },
|
||||
'/_app/version.json': { url: 'https://example.org/public-asset' },
|
||||
'/a/b': { url: 'https://example.org/decoded' },
|
||||
'/a%2Fb': { url: 'https://example.org/literal-percent' },
|
||||
'/docs': { url: 'https://example.org/first-domain' },
|
||||
'/healthz': { url: 'https://example.org/public-health' }
|
||||
}
|
||||
},
|
||||
'tiny.example.org': {
|
||||
links: { '/docs': { url: 'https://example.org/second-domain' } }
|
||||
const configuration = {
|
||||
site: { url: `http://localhost:${address.port}` },
|
||||
auth: { credentials: { username: 'example', password: 'e2e-only-password' } },
|
||||
domains: {
|
||||
'go.example.org': {
|
||||
links: {
|
||||
'/schema.json': { url: 'https://example.org/public-schema' },
|
||||
'/_app/version.json': { url: 'https://example.org/public-asset' },
|
||||
'/a/b': { url: 'https://example.org/decoded' },
|
||||
'/a%2Fb': { url: 'https://example.org/literal-percent' },
|
||||
'/docs': { url: 'https://example.org/first-domain' },
|
||||
'/healthz': { url: 'https://example.org/public-health' }
|
||||
}
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
},
|
||||
'tiny.example.org': { links: { '/docs': { url: 'https://example.org/second-domain' } } }
|
||||
}
|
||||
};
|
||||
await writeFile(join(runtimeDirectory, 'config/goto.yaml'), JSON.stringify(configuration));
|
||||
if (process.env.GOTO_E2E_WORKER === '1') {
|
||||
await seedVisits();
|
||||
await startWorker();
|
||||
}
|
||||
await import(pathToFileURL(join(directory, 'server.js')).href);
|
||||
|
||||
async function seedVisits() {
|
||||
const db = await openDb(join(runtimeDirectory, 'data/goto.sqlite'));
|
||||
try {
|
||||
await runMigrations(db);
|
||||
const common = {
|
||||
visited_at: Date.now() - 1000,
|
||||
ip: '203.0.113.1',
|
||||
user_agent: null,
|
||||
accept_language: null,
|
||||
browser: 'Other',
|
||||
os: 'Other'
|
||||
};
|
||||
const visits = [
|
||||
{ domain: 'go.example.org', path: '/docs', outcome: 'valid' },
|
||||
{ domain: 'go.example.org', path: '/a/b', outcome: 'invalid' },
|
||||
{ domain: 'go.example.org', path: '/', outcome: 'invalid' },
|
||||
{ domain: 'go.example.org', path: '/a%2Fb', outcome: 'valid' },
|
||||
{ domain: 'tiny.example.org', path: '/docs', outcome: 'valid' }
|
||||
];
|
||||
for (const visit of visits) await insertVisit(db, { ...common, ...visit });
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
async function startWorker() {
|
||||
const worker = fork(new URL('../../worker/dist/index.js', import.meta.url), [], {
|
||||
cwd: runtimeDirectory,
|
||||
env: { ...process.env, HOST: '127.0.0.1', PORT: '0', SHUTDOWN_TIMEOUT: '5' },
|
||||
execArgv: [],
|
||||
stdio: ['ignore', 'inherit', 'inherit', 'ipc']
|
||||
});
|
||||
process.on('SIGTERM', () => worker.kill('SIGTERM'));
|
||||
process.on('SIGINT', () => worker.kill('SIGTERM'));
|
||||
process.on('disconnect', () => worker.kill('SIGTERM'));
|
||||
process.once('exit', () => worker.kill('SIGKILL'));
|
||||
worker.once('exit', (code, signal) => {
|
||||
if (code !== 0 || signal !== null) process.exitCode = 1;
|
||||
});
|
||||
const ready = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
worker.kill('SIGKILL');
|
||||
reject(new Error('E2E worker startup timed out'));
|
||||
}, 15_000);
|
||||
worker.once('message', (message) => {
|
||||
clearTimeout(timer);
|
||||
resolve(message);
|
||||
});
|
||||
worker.once('error', (error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
});
|
||||
worker.once('exit', () => {
|
||||
clearTimeout(timer);
|
||||
reject(new Error('E2E worker exited before readiness'));
|
||||
});
|
||||
});
|
||||
if (ready.type !== 'ready') throw new Error('Invalid worker readiness');
|
||||
}
|
||||
@@ -53,13 +53,17 @@ async function stopServer(child: ChildProcess): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
export const test = base.extend<{ appServer: AppServer }>({
|
||||
// eslint-disable-next-line no-empty-pattern -- Playwright discovers dependencies by destructuring.
|
||||
appServer: async ({}, use, testInfo) => {
|
||||
export const test = base.extend<{ appServer: AppServer; withWorker: boolean }>({
|
||||
withWorker: [false, { option: true }],
|
||||
appServer: async ({ withWorker }, use, testInfo) => {
|
||||
const directory = process.env.GOTO_E2E_BUILD_DIR;
|
||||
if (!directory) throw new Error('Missing E2E build directory');
|
||||
const child = fork(new URL('../server.mjs', import.meta.url), [], {
|
||||
env: { ...serverEnvironment(), GOTO_E2E_BUILD_DIR: directory },
|
||||
env: {
|
||||
...serverEnvironment(),
|
||||
GOTO_E2E_BUILD_DIR: directory,
|
||||
GOTO_E2E_WORKER: withWorker ? '1' : '0'
|
||||
},
|
||||
execArgv: [],
|
||||
stdio: ['ignore', 'pipe', 'pipe', 'ipc']
|
||||
});
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
"format": "prettier --ignore-path ../../.prettierignore --write .",
|
||||
"test": "vitest run",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"test:e2e": "playwright test"
|
||||
"test:e2e": "pnpm --filter @goto/worker... build && playwright test"
|
||||
},
|
||||
"dependencies": {
|
||||
"@goto/config": "workspace:*",
|
||||
|
||||
+3
-2
@@ -2,7 +2,7 @@
|
||||
import { page } from '$app/state';
|
||||
import { goto } from '$app/navigation';
|
||||
import { authClient } from '#lib/auth-client.js';
|
||||
import ThemeToggle from './ThemeToggle.svelte';
|
||||
import ThemeToggle from '../ThemeToggle.svelte';
|
||||
|
||||
let { title }: { title: string } = $props();
|
||||
|
||||
@@ -49,7 +49,8 @@
|
||||
<header class="admin-nav">
|
||||
<div class="nav-container">
|
||||
<nav aria-label="Administration">
|
||||
<a href="/admin" aria-current="page">{title}</a>
|
||||
<a href="/admin" aria-current={page.url.pathname === '/admin' ? 'page' : undefined}>{title}</a
|
||||
>
|
||||
</nav>
|
||||
<div class="nav-actions">
|
||||
<ThemeToggle />
|
||||
@@ -0,0 +1,39 @@
|
||||
<script lang="ts">
|
||||
import type { Snippet } from 'svelte';
|
||||
import AdminNav from './AdminNav.svelte';
|
||||
import { formatUpdated } from './format.js';
|
||||
import '#lib/styles/admin.css';
|
||||
|
||||
let {
|
||||
siteTitle,
|
||||
title,
|
||||
updatedAt = null,
|
||||
backHref = '',
|
||||
backLabel = 'Overview',
|
||||
children
|
||||
}: {
|
||||
siteTitle: string;
|
||||
title: string;
|
||||
updatedAt?: number | null;
|
||||
backHref?: string;
|
||||
backLabel?: string;
|
||||
children: Snippet;
|
||||
} = $props();
|
||||
</script>
|
||||
|
||||
<svelte:head><title>{title} — {siteTitle}</title></svelte:head>
|
||||
<AdminNav title={siteTitle} />
|
||||
<main class="admin-page">
|
||||
<header class="admin-heading">
|
||||
{#if backHref}<nav aria-label="Breadcrumb"><a href={backHref}>{backLabel}</a></nav>{/if}
|
||||
<div class="admin-title-row">
|
||||
<h1>{title}</h1>
|
||||
{#if updatedAt !== null}
|
||||
<time class="admin-updated" datetime={new Date(updatedAt).toISOString()}
|
||||
>{formatUpdated(updatedAt)}</time
|
||||
>
|
||||
{/if}
|
||||
</div>
|
||||
</header>
|
||||
{@render children()}
|
||||
</main>
|
||||
@@ -0,0 +1,10 @@
|
||||
<script lang="ts">
|
||||
let { collectionEnabled, available }: { collectionEnabled: boolean; available: boolean } =
|
||||
$props();
|
||||
</script>
|
||||
|
||||
{#if !collectionEnabled}
|
||||
<p class="admin-notice" role="status">Analytics disabled.</p>
|
||||
{:else if !available}
|
||||
<p class="admin-notice" role="status">Statistics unavailable.</p>
|
||||
{/if}
|
||||
@@ -0,0 +1,9 @@
|
||||
<script lang="ts">
|
||||
import { formatCount } from './format.js';
|
||||
let { label, value }: { label: string; value: number | string | null } = $props();
|
||||
</script>
|
||||
|
||||
<section class="admin-stat" aria-label={label}>
|
||||
<span class="admin-stat-value">{typeof value === 'string' ? value : formatCount(value)}</span>
|
||||
<span class="admin-stat-label">{label}</span>
|
||||
</section>
|
||||
@@ -0,0 +1,13 @@
|
||||
import { expect, test } from 'vitest';
|
||||
import { formatUpdated, formatRetention } from './format.js';
|
||||
|
||||
test('formats the snapshot update time in UTC', () => {
|
||||
const updatedAt = Date.parse('2026-11-06T22:03:00Z');
|
||||
expect(formatUpdated(updatedAt)).toBe('Updated Nov 6, 2026, 10:03 PM UTC');
|
||||
expect(formatUpdated(null)).toBe('');
|
||||
});
|
||||
|
||||
test('shows the configured retention policy rather than an elapsed day count', () => {
|
||||
expect(formatRetention(3)).toBe('3 months');
|
||||
expect(formatRetention(0)).toBe('Off');
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
export function formatCount(value: number | null): string {
|
||||
return value === null ? '—' : value.toLocaleString('en');
|
||||
}
|
||||
|
||||
export function formatUpdated(value: number | null): string {
|
||||
if (value === null) return '';
|
||||
const formatter = new Intl.DateTimeFormat('en', {
|
||||
dateStyle: 'medium',
|
||||
timeStyle: 'short',
|
||||
timeZone: 'UTC'
|
||||
});
|
||||
return `Updated ${formatter.format(value)} UTC`;
|
||||
}
|
||||
|
||||
export function formatRetention(months: number): string {
|
||||
return months === 0 ? 'Off' : `${months} months`;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { PreparedLink } from '@goto/config';
|
||||
import type { Immutable } from './snapshot.js';
|
||||
|
||||
export function currentLinkState(link: Immutable<PreparedLink>, now: number): string {
|
||||
if (!link.enabled) return 'Disabled';
|
||||
if (link.valid_from !== undefined && now < Date.parse(link.valid_from)) return 'Scheduled';
|
||||
if (link.valid_until !== undefined && now >= Date.parse(link.valid_until)) return 'Expired';
|
||||
return 'Active';
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { access, mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { GotoConfiguration } from '@goto/config';
|
||||
import { openDb, runMigrations } from '@goto/db';
|
||||
import { publishVisitReports, type PreparedVisitReport } from '@goto/db/visit-reports';
|
||||
import { expect, test, vi } from 'vitest';
|
||||
import { readVisitReport } from './visit-reports.js';
|
||||
|
||||
const report: PreparedVisitReport = {
|
||||
totals: { total: 3, valid: 2, invalid: 1, base: 1 },
|
||||
daily: [],
|
||||
weekly: [],
|
||||
monthly: [],
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [],
|
||||
linkCounts: []
|
||||
};
|
||||
|
||||
test('reads prepared output and hides snapshots from a different retention policy', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goto-page-reports-'));
|
||||
const configuration: Pick<GotoConfiguration, 'database' | 'analytics'> = {
|
||||
database: {
|
||||
path: join(directory, 'goto.sqlite'),
|
||||
worker_path: join(directory, 'worker.sqlite')
|
||||
},
|
||||
analytics: { retention_months: 3 }
|
||||
};
|
||||
try {
|
||||
const db = await openDb(configuration.database.path);
|
||||
try {
|
||||
await runMigrations(db);
|
||||
const period = {
|
||||
timezone: 'UTC' as const,
|
||||
retentionMonths: 3 as const,
|
||||
since: 100,
|
||||
asOf: 200,
|
||||
days: [],
|
||||
weeks: [],
|
||||
months: []
|
||||
};
|
||||
await publishVisitReports(db, period, [
|
||||
{ key: 'overview', data: JSON.stringify(report) },
|
||||
{ key: 'empty', data: JSON.stringify(report) }
|
||||
]);
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
expect((await readVisitReport(configuration))?.report.totals.total).toBe(3);
|
||||
configuration.analytics.retention_months = 9;
|
||||
expect(await readVisitReport(configuration)).toBeNull();
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('enabled collection fails safely without creating missing storage', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goto-missing-reports-'));
|
||||
const missing = join(directory, 'missing');
|
||||
const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const configuration = {
|
||||
database: {
|
||||
path: join(missing, 'goto.sqlite'),
|
||||
worker_path: join(directory, 'worker.sqlite')
|
||||
},
|
||||
analytics: { retention_months: 3 as const }
|
||||
};
|
||||
expect(await readVisitReport(configuration)).toBeNull();
|
||||
expect(warning).toHaveBeenCalledExactlyOnceWith('Prepared visit reports unavailable');
|
||||
await expect(access(missing)).rejects.toMatchObject({ code: 'ENOENT' });
|
||||
} finally {
|
||||
warning.mockRestore();
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('disabled collection does not attempt to open missing storage', async () => {
|
||||
const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
try {
|
||||
const configuration = {
|
||||
database: { path: '/missing/not-used.sqlite', worker_path: '/missing/worker.sqlite' },
|
||||
analytics: { retention_months: 0 as const }
|
||||
};
|
||||
expect(await readVisitReport(configuration)).toBeNull();
|
||||
expect(warning).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
warning.mockRestore();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { GotoConfiguration } from '@goto/config';
|
||||
import { openDb } from '@goto/db';
|
||||
import { readPreparedVisitReports, type StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import type { Immutable } from './snapshot.js';
|
||||
|
||||
type ReportConfiguration = Immutable<Pick<GotoConfiguration, 'database' | 'analytics'>>;
|
||||
|
||||
export async function readVisitReport(
|
||||
configuration: ReportConfiguration,
|
||||
domain?: string,
|
||||
path?: string
|
||||
): Promise<StoredVisitReports | null> {
|
||||
const months = configuration.analytics.retention_months;
|
||||
if (months === 0) return null;
|
||||
let db: Awaited<ReturnType<typeof openDb>> | undefined;
|
||||
try {
|
||||
db = await openDb(configuration.database.path, { readOnly: true });
|
||||
const reports = await readPreparedVisitReports(db, domain, path);
|
||||
if (!reports || reports.period.retentionMonths !== months) return null;
|
||||
return reports;
|
||||
} catch {
|
||||
warnUnavailable();
|
||||
return null;
|
||||
} finally {
|
||||
await db?.destroy().catch(warnUnavailable);
|
||||
}
|
||||
}
|
||||
|
||||
function warnUnavailable(): void {
|
||||
try {
|
||||
console.warn('Prepared visit reports unavailable');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
.admin-page {
|
||||
max-width: var(--content-admin-width);
|
||||
margin-inline: auto;
|
||||
padding: var(--space-8) var(--space-5);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-8);
|
||||
}
|
||||
.admin-heading {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
.admin-title-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
align-items: baseline;
|
||||
gap: var(--space-4) var(--space-5);
|
||||
}
|
||||
.admin-updated {
|
||||
margin-inline-start: auto;
|
||||
text-align: end;
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
.admin-heading h1 {
|
||||
font-size: var(--font-size-heading);
|
||||
line-height: var(--line-height-heading);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.admin-page h2 {
|
||||
font-size: 1.25rem;
|
||||
margin-bottom: var(--space-4);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.admin-page a {
|
||||
color: var(--color-primary);
|
||||
}
|
||||
.admin-cards {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
||||
gap: var(--space-5);
|
||||
}
|
||||
.admin-cards-summary {
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
}
|
||||
.admin-stat,
|
||||
.admin-info {
|
||||
background: var(--color-surface-card);
|
||||
border: var(--border-width) solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-card);
|
||||
padding: var(--space-5);
|
||||
}
|
||||
.admin-stat {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
.admin-stat-value {
|
||||
font-size: var(--font-size-heading);
|
||||
font-weight: 700;
|
||||
}
|
||||
.admin-stat-label {
|
||||
font-weight: 600;
|
||||
}
|
||||
.admin-notice {
|
||||
padding: var(--space-5);
|
||||
border: var(--border-width) solid var(--color-border-strong);
|
||||
border-radius: var(--radius-card);
|
||||
}
|
||||
.admin-table-scroll {
|
||||
overflow-x: auto;
|
||||
border: var(--border-width) solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-card);
|
||||
background: var(--color-surface-card);
|
||||
}
|
||||
.admin-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: var(--font-size-sm);
|
||||
}
|
||||
.admin-table th,
|
||||
.admin-table td {
|
||||
padding: var(--space-4);
|
||||
text-align: start;
|
||||
vertical-align: top;
|
||||
border-top: var(--border-width) solid var(--color-border-subtle);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.admin-table thead {
|
||||
background: var(--color-surface-muted);
|
||||
}
|
||||
.admin-link-table {
|
||||
table-layout: fixed;
|
||||
min-width: 42rem;
|
||||
}
|
||||
.admin-link-table thead th:nth-child(1) {
|
||||
width: 18%;
|
||||
}
|
||||
.admin-link-table thead th:nth-child(2) {
|
||||
width: 46%;
|
||||
}
|
||||
.admin-link-table thead th:nth-child(3) {
|
||||
width: 20%;
|
||||
}
|
||||
.admin-link-table thead th:nth-child(4) {
|
||||
width: 16%;
|
||||
}
|
||||
.admin-table tbody tr {
|
||||
position: relative;
|
||||
}
|
||||
.admin-table tbody tr:hover,
|
||||
.admin-table tbody tr:focus-within {
|
||||
background: var(--color-surface-muted);
|
||||
}
|
||||
.admin-row-link::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
}
|
||||
.admin-row-link:focus-visible {
|
||||
outline: none;
|
||||
}
|
||||
.admin-row-link:focus-visible::after {
|
||||
outline: var(--outline-focus);
|
||||
outline-offset: -2px;
|
||||
}
|
||||
.admin-destination {
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
}
|
||||
.admin-info dl {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(7rem, auto) minmax(0, 1fr);
|
||||
gap: var(--space-4) var(--space-5);
|
||||
}
|
||||
.admin-info dt {
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
.admin-info dd {
|
||||
margin: 0;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
@media (max-width: 640px) {
|
||||
.admin-cards-summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
.admin-info dl {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@media (max-width: 400px) {
|
||||
.admin-cards-summary {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,31 @@
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import type { StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = ({ locals }) => ({
|
||||
domains: Object.entries(locals.snapshot.domains).map(([domain, prepared]) => ({
|
||||
export const load: PageServerLoad = async ({ locals }) => {
|
||||
const snapshot = locals.snapshot;
|
||||
const configuration = snapshot.configuration;
|
||||
const retentionMonths = configuration.analytics.retention_months;
|
||||
const collectionEnabled = retentionMonths !== 0;
|
||||
let preparedReports: StoredVisitReports | null = null;
|
||||
if (collectionEnabled) preparedReports = await readVisitReport(configuration);
|
||||
const report = preparedReports?.report;
|
||||
|
||||
const domains = Object.entries(snapshot.domains).map(([domain, prepared]) => ({
|
||||
domain,
|
||||
links: Object.entries(prepared.links).map(([path, link]) => ({
|
||||
path,
|
||||
title: link.title,
|
||||
url: link.url,
|
||||
enabled: link.enabled,
|
||||
status_code: link.status_code
|
||||
}))
|
||||
}))
|
||||
});
|
||||
linkCount: Object.keys(prepared.links).length,
|
||||
counts: report?.domainCounts.find((row) => row.domain === domain)?.counts ?? null,
|
||||
href: `/admin/domains/${encodeURIComponent(domain)}`
|
||||
}));
|
||||
return {
|
||||
collectionEnabled,
|
||||
available: preparedReports !== null,
|
||||
retentionMonths,
|
||||
since: preparedReports?.period.since ?? null,
|
||||
asOf: preparedReports?.period.asOf ?? null,
|
||||
domainCount: domains.length,
|
||||
linkCount: domains.reduce((sum, row) => sum + row.linkCount, 0),
|
||||
totals: report?.totals ?? null,
|
||||
domains
|
||||
};
|
||||
};
|
||||
@@ -1,127 +1,42 @@
|
||||
<script lang="ts">
|
||||
import AdminNav from '#lib/components/AdminNav.svelte';
|
||||
import AdminPage from '#lib/components/admin/AdminPage.svelte';
|
||||
import StatCard from '#lib/components/admin/StatCard.svelte';
|
||||
import ReportNotice from '#lib/components/admin/ReportNotice.svelte';
|
||||
import { formatCount, formatRetention } from '#lib/components/admin/format.js';
|
||||
import type { PageData } from './$types';
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
<title>Configured links — {data.title}</title>
|
||||
</svelte:head>
|
||||
|
||||
<AdminNav title={data.title} />
|
||||
|
||||
<main>
|
||||
<header class="page-header">
|
||||
<h1>Configured links</h1>
|
||||
<p>Read-only. Edit the YAML configuration to change links.</p>
|
||||
</header>
|
||||
|
||||
{#if data.domains.length === 0}
|
||||
<p>No links configured.</p>
|
||||
{:else}
|
||||
{#each data.domains as group (group.domain)}
|
||||
<section>
|
||||
<h2>{group.domain}</h2>
|
||||
<div class="table-scroll" role="region" aria-label={`Links on ${group.domain}`}>
|
||||
<table>
|
||||
<caption>Configured links on {group.domain}</caption>
|
||||
<thead>
|
||||
<AdminPage siteTitle={data.title} title="Overview" updatedAt={data.asOf}>
|
||||
<ReportNotice collectionEnabled={data.collectionEnabled} available={data.available} />
|
||||
<div class="admin-cards admin-cards-summary">
|
||||
<StatCard label="Domains" value={data.domainCount} />
|
||||
<StatCard label="Links" value={data.linkCount} />
|
||||
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
|
||||
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
|
||||
</div>
|
||||
<section>
|
||||
<h2>Domains</h2>
|
||||
{#if data.domains.length === 0}<p>No domains configured.</p>
|
||||
{:else}
|
||||
<div class="admin-table-scroll" role="region" aria-label="Domain inventory">
|
||||
<table class="admin-table">
|
||||
<thead
|
||||
><tr
|
||||
><th scope="col">Domain</th><th scope="col">Links</th><th scope="col">Visits</th></tr
|
||||
></thead
|
||||
>
|
||||
<tbody>
|
||||
{#each data.domains as row (row.domain)}
|
||||
<tr>
|
||||
<th scope="col">Path</th>
|
||||
<th scope="col">Title</th>
|
||||
<th scope="col">Destination</th>
|
||||
<th scope="col">Enabled</th>
|
||||
<th scope="col">HTTP status</th>
|
||||
<th scope="row"><a class="admin-row-link" href={row.href}>{row.domain}</a></th>
|
||||
<td>{formatCount(row.linkCount)}</td>
|
||||
<td>{formatCount(row.counts?.valid ?? null)}</td>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each group.links as link (link.path)}
|
||||
<tr>
|
||||
<th scope="row"><code>{link.path}</code></th>
|
||||
<td>{link.title}</td>
|
||||
<td><a href={link.url}>{link.url}</a></td>
|
||||
<td>{link.enabled ? 'Yes' : 'No'}</td>
|
||||
<td>{link.status_code}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
{/each}
|
||||
{/if}
|
||||
</main>
|
||||
|
||||
<style>
|
||||
main {
|
||||
max-width: var(--content-admin-width);
|
||||
margin-inline: auto;
|
||||
padding: var(--space-8) var(--space-5);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-8);
|
||||
}
|
||||
|
||||
.page-header {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: var(--space-4);
|
||||
}
|
||||
|
||||
h1 {
|
||||
font-size: var(--font-size-heading);
|
||||
line-height: var(--line-height-heading);
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-size: var(--font-size-base);
|
||||
overflow-wrap: anywhere;
|
||||
margin-bottom: var(--space-4);
|
||||
}
|
||||
|
||||
.page-header p,
|
||||
caption {
|
||||
color: var(--color-text-muted);
|
||||
}
|
||||
|
||||
.table-scroll {
|
||||
overflow-x: auto;
|
||||
border: var(--border-width) solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-card);
|
||||
background: var(--color-surface-card);
|
||||
}
|
||||
|
||||
.table-scroll:focus-visible {
|
||||
outline: var(--outline-focus);
|
||||
outline-offset: var(--space-1);
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: var(--font-size-sm);
|
||||
}
|
||||
|
||||
caption {
|
||||
padding: var(--space-4);
|
||||
text-align: start;
|
||||
}
|
||||
|
||||
th,
|
||||
td {
|
||||
padding: var(--space-4);
|
||||
text-align: start;
|
||||
vertical-align: top;
|
||||
border-top: var(--border-width) solid var(--color-border-subtle);
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
thead {
|
||||
background: var(--color-surface-muted);
|
||||
}
|
||||
|
||||
a {
|
||||
color: var(--color-primary);
|
||||
}
|
||||
</style>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
</AdminPage>
|
||||
@@ -0,0 +1,64 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { VisitCounts, StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import { currentLinkState } from '#lib/server/link-state.js';
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = async ({ locals, params }) => {
|
||||
const snapshot = locals.snapshot;
|
||||
const domain = params.domain;
|
||||
if (!Object.hasOwn(snapshot.domains, domain)) throw error(404, 'Domain not configured');
|
||||
const prepared = snapshot.domains[domain]!;
|
||||
const configuration = snapshot.configuration;
|
||||
const collectionEnabled = configuration.analytics.retention_months !== 0;
|
||||
let preparedReports: StoredVisitReports | null = null;
|
||||
if (collectionEnabled) preparedReports = await readVisitReport(configuration, domain);
|
||||
const report = preparedReports?.report;
|
||||
|
||||
const countsByPath = new Map<string, VisitCounts>();
|
||||
for (const row of report?.linkCounts ?? []) countsByPath.set(row.path, row.counts);
|
||||
const baseCounts = countsByPath.get('/') ?? null;
|
||||
let invalidClicks: number | null = null;
|
||||
if (report && baseCounts) invalidClicks = report.totals.invalid - baseCounts.invalid;
|
||||
const now = Date.now();
|
||||
const links = Object.entries(prepared.links).map(([path, link]) => ({
|
||||
path,
|
||||
url: link.url,
|
||||
state: currentLinkState(link, now),
|
||||
counts: countsByPath.get(path) ?? null,
|
||||
href: linkHref(domain, path)
|
||||
}));
|
||||
const root = prepared.links['/'];
|
||||
let baseUrl = prepared.fallback === 404 ? null : prepared.fallback;
|
||||
let baseState = 'Fallback';
|
||||
if (root && currentLinkState(root, now) === 'Active') {
|
||||
baseUrl = root.url;
|
||||
baseState = 'Active';
|
||||
}
|
||||
|
||||
return {
|
||||
domain,
|
||||
retentionMonths: configuration.analytics.retention_months,
|
||||
collectionEnabled,
|
||||
available: preparedReports !== null,
|
||||
since: preparedReports?.period.since ?? null,
|
||||
asOf: preparedReports?.period.asOf ?? null,
|
||||
linkCount: links.length,
|
||||
totals: report?.totals ?? null,
|
||||
invalidClicks,
|
||||
base: {
|
||||
url: baseUrl,
|
||||
state: baseState,
|
||||
counts: baseCounts,
|
||||
href: linkHref(domain, '/')
|
||||
},
|
||||
links
|
||||
};
|
||||
};
|
||||
|
||||
function linkHref(domain: string, path: string): string {
|
||||
const base = `/admin/domains/${encodeURIComponent(domain)}/links`;
|
||||
if (path === '/') return base;
|
||||
const segments = path.slice(1).split('/').map(encodeURIComponent);
|
||||
return `${base}/${segments.join('/')}`;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
<script lang="ts">
|
||||
import AdminPage from '#lib/components/admin/AdminPage.svelte';
|
||||
import StatCard from '#lib/components/admin/StatCard.svelte';
|
||||
import ReportNotice from '#lib/components/admin/ReportNotice.svelte';
|
||||
import { formatCount, formatRetention } from '#lib/components/admin/format.js';
|
||||
import type { PageData } from './$types';
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
|
||||
<AdminPage siteTitle={data.title} title={data.domain} updatedAt={data.asOf} backHref="/admin">
|
||||
<ReportNotice collectionEnabled={data.collectionEnabled} available={data.available} />
|
||||
<div class="admin-cards admin-cards-summary">
|
||||
<StatCard label="Links" value={data.linkCount} />
|
||||
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
|
||||
<StatCard label="Invalid clicks" value={data.invalidClicks} />
|
||||
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
|
||||
</div>
|
||||
<section>
|
||||
<h2>Base URL</h2>
|
||||
<div class="admin-table-scroll" role="region" aria-label={`Base URL on ${data.domain}`}>
|
||||
<table class="admin-table admin-link-table">
|
||||
<thead
|
||||
><tr
|
||||
><th scope="col">Path</th><th scope="col">Destination</th><th scope="col">State</th><th
|
||||
scope="col">Visits</th
|
||||
></tr
|
||||
></thead
|
||||
>
|
||||
<tbody>
|
||||
<tr>
|
||||
<th scope="row"><a class="admin-row-link" href={data.base.href}><code>/</code></a></th>
|
||||
<td
|
||||
>{#if data.base.url}<a class="admin-destination" href={data.base.url}
|
||||
>{data.base.url}</a
|
||||
>{:else}404 Not found{/if}</td
|
||||
>
|
||||
<td>{data.base.state}</td>
|
||||
<td>{formatCount(data.base.counts?.valid ?? null)}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Links</h2>
|
||||
{#if data.links.length === 0}<p>No links configured.</p>
|
||||
{:else}
|
||||
<div class="admin-table-scroll" role="region" aria-label={`Links on ${data.domain}`}>
|
||||
<table class="admin-table admin-link-table">
|
||||
<thead
|
||||
><tr
|
||||
><th scope="col">Path</th><th scope="col">Destination</th><th scope="col">State</th
|
||||
><th scope="col">Visits</th></tr
|
||||
></thead
|
||||
>
|
||||
<tbody>
|
||||
{#each data.links as link (link.path)}
|
||||
<tr>
|
||||
<th scope="row"
|
||||
><a class="admin-row-link" href={link.href}><code>{link.path}</code></a></th
|
||||
>
|
||||
<td><a class="admin-destination" href={link.url}>{link.url}</a></td>
|
||||
<td>{link.state}</td><td>{formatCount(link.counts?.valid ?? null)}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{/if}
|
||||
</section>
|
||||
</AdminPage>
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { error } from '@sveltejs/kit';
|
||||
import type { StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import { currentLinkState } from '#lib/server/link-state.js';
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import type { PageServerLoad } from './$types';
|
||||
|
||||
export const load: PageServerLoad = async ({ locals, params }) => {
|
||||
const snapshot = locals.snapshot;
|
||||
const domain = params.domain;
|
||||
if (!Object.hasOwn(snapshot.domains, domain)) throw error(404, 'Domain not configured');
|
||||
const prepared = snapshot.domains[domain]!;
|
||||
// Kit has decoded route parameters; preserve literal percent paths as-is.
|
||||
const path = `/${params.path ?? ''}`;
|
||||
if (path !== '/' && !Object.hasOwn(prepared.links, path)) throw error(404, 'Link not configured');
|
||||
const link = prepared.links[path];
|
||||
const configuration = snapshot.configuration;
|
||||
const collectionEnabled = configuration.analytics.retention_months !== 0;
|
||||
let reports: StoredVisitReports | null = null;
|
||||
if (collectionEnabled) reports = await readVisitReport(configuration, domain, path);
|
||||
const state = link ? currentLinkState(link, Date.now()) : 'Fallback';
|
||||
|
||||
return {
|
||||
domain,
|
||||
path,
|
||||
domainHref: `/admin/domains/${encodeURIComponent(domain)}`,
|
||||
url: link?.url ?? null,
|
||||
fallback: prepared.fallback,
|
||||
state,
|
||||
retentionMonths: configuration.analytics.retention_months,
|
||||
collectionEnabled,
|
||||
available: reports !== null,
|
||||
since: reports?.period.since ?? null,
|
||||
asOf: reports?.period.asOf ?? null,
|
||||
totals: reports?.selection.totals ?? null
|
||||
};
|
||||
};
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
<script lang="ts">
|
||||
import AdminPage from '#lib/components/admin/AdminPage.svelte';
|
||||
import StatCard from '#lib/components/admin/StatCard.svelte';
|
||||
import ReportNotice from '#lib/components/admin/ReportNotice.svelte';
|
||||
import { formatRetention } from '#lib/components/admin/format.js';
|
||||
import type { PageData } from './$types';
|
||||
let { data }: { data: PageData } = $props();
|
||||
</script>
|
||||
|
||||
<AdminPage
|
||||
siteTitle={data.title}
|
||||
title={data.path}
|
||||
updatedAt={data.asOf}
|
||||
backHref={data.domainHref}
|
||||
backLabel={data.domain}
|
||||
>
|
||||
<ReportNotice collectionEnabled={data.collectionEnabled} available={data.available} />
|
||||
<div class="admin-cards">
|
||||
<StatCard label="Valid visits" value={data.totals?.valid ?? null} />
|
||||
<StatCard label="Retention" value={formatRetention(data.retentionMonths)} />
|
||||
</div>
|
||||
<section class="admin-info" aria-label="Link details">
|
||||
<dl>
|
||||
<dt>Destination</dt>
|
||||
<dd>
|
||||
{#if data.url}<a class="admin-destination" href={data.url}>{data.url}</a>
|
||||
{:else if data.fallback === 404}404 Not found{:else}{data.fallback}{/if}
|
||||
</dd>
|
||||
<dt>State</dt>
|
||||
<dd>{data.state}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
</AdminPage>
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import type { PreparedVisitReport } from '@goto/db/visit-reports';
|
||||
import { beforeEach, expect, test, vi } from 'vitest';
|
||||
import { prepareSnapshot } from '#lib/server/snapshot.js';
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import { load } from './+page.server.js';
|
||||
|
||||
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
const domain = 'go.example.test';
|
||||
function request(path = '/docs', host = domain, retention_months = 3) {
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'link-test-password' } },
|
||||
analytics: { retention_months },
|
||||
domains: {
|
||||
[domain]: {
|
||||
links: {
|
||||
'/docs': { url: 'https://example.test/docs' },
|
||||
'/a/b': { url: 'https://example.test/nested' },
|
||||
'/a%2Fb': { url: 'https://example.test/percent', enabled: false }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
return {
|
||||
locals: { snapshot: prepareSnapshot(prepared) },
|
||||
params: { domain: host, path: path.slice(1) }
|
||||
} as Parameters<typeof load>[0];
|
||||
}
|
||||
const report: PreparedVisitReport = {
|
||||
totals: { total: 6, valid: 3, invalid: 3, base: 1 },
|
||||
daily: [],
|
||||
weekly: [],
|
||||
monthly: [],
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [],
|
||||
linkCounts: []
|
||||
};
|
||||
const period = {
|
||||
timezone: 'UTC' as const,
|
||||
retentionMonths: 3 as const,
|
||||
since: 100,
|
||||
asOf: 200,
|
||||
days: [],
|
||||
weeks: [],
|
||||
months: []
|
||||
};
|
||||
|
||||
test('loads the requested path report and current destination with a domain breadcrumb', async () => {
|
||||
const totals = { total: 3, valid: 2, invalid: 1, base: 0 };
|
||||
vi.mocked(readVisitReport).mockResolvedValue({
|
||||
period,
|
||||
report,
|
||||
selection: { ...report, totals }
|
||||
});
|
||||
const data = await load(request());
|
||||
if (!data) throw new Error('Missing link page');
|
||||
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/docs');
|
||||
expect(data.path).toBe('/docs');
|
||||
expect(data.totals).toEqual(totals);
|
||||
expect(data.url).toBe('https://example.test/docs');
|
||||
expect(data.state).toBe('Active');
|
||||
expect(data.domainHref).toBe('/admin/domains/go.example.test');
|
||||
expect(data.asOf).toBe(200);
|
||||
expect(data.retentionMonths).toBe(3);
|
||||
});
|
||||
|
||||
test('keeps decoded nested and literal-percent paths distinct and supports the unconfigured root', async () => {
|
||||
vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report });
|
||||
const nested = await load(request('/a/b'));
|
||||
const percent = await load(request('/a%2Fb'));
|
||||
const root = await load(request('/'));
|
||||
if (!nested || !percent || !root) throw new Error('Missing path page');
|
||||
expect(nested.path).toBe('/a/b');
|
||||
expect(percent.path).toBe('/a%2Fb');
|
||||
expect(percent.state).toBe('Disabled');
|
||||
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/a%2Fb');
|
||||
expect(root.path).toBe('/');
|
||||
expect(root.state).toBe('Fallback');
|
||||
expect(root.url).toBeNull();
|
||||
expect(root.fallback).toBe(404);
|
||||
});
|
||||
|
||||
test('rejects unknown scopes before reading and keeps unavailable or disabled counts empty', async () => {
|
||||
await expect(load(request('/docs', 'unknown.test'))).rejects.toMatchObject({ status: 404 });
|
||||
await expect(load(request('/not-configured'))).rejects.toMatchObject({ status: 404 });
|
||||
expect(readVisitReport).not.toHaveBeenCalled();
|
||||
vi.mocked(readVisitReport).mockResolvedValue(null);
|
||||
const missing = await load(request());
|
||||
if (!missing) throw new Error('Missing unavailable page');
|
||||
expect(missing.available).toBe(false);
|
||||
expect(missing.totals).toBeNull();
|
||||
vi.clearAllMocks();
|
||||
const disabled = await load(request('/docs', domain, 0));
|
||||
if (!disabled) throw new Error('Missing disabled page');
|
||||
expect(disabled.collectionEnabled).toBe(false);
|
||||
expect(disabled.retentionMonths).toBe(0);
|
||||
expect(disabled.totals).toBeNull();
|
||||
expect(readVisitReport).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import type { PreparedVisitReport, StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import { beforeEach, expect, test, vi } from 'vitest';
|
||||
import { prepareSnapshot } from '#lib/server/snapshot.js';
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import { load } from './+page.server.js';
|
||||
|
||||
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
const domain = 'go.example.test';
|
||||
function request(host = domain, rootUrl?: string) {
|
||||
const links: Record<string, { url: string; enabled?: boolean }> = {
|
||||
'/docs': { url: 'https://example.test/docs' },
|
||||
'/a/b': { url: 'https://example.test/nested' },
|
||||
'/a%2Fb': { url: 'https://example.test/percent', enabled: false }
|
||||
};
|
||||
if (rootUrl) links['/'] = { url: rootUrl };
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'domain-test-password' } },
|
||||
domains: { [domain]: { links } }
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
return {
|
||||
locals: { snapshot: prepareSnapshot(prepared) },
|
||||
params: { domain: host }
|
||||
} as Parameters<typeof load>[0];
|
||||
}
|
||||
const docs = { total: 3, valid: 2, invalid: 1, base: 0 };
|
||||
const root = { total: 1, valid: 0, invalid: 1, base: 1 };
|
||||
const empty = { total: 0, valid: 0, invalid: 0, base: 0 };
|
||||
const report: PreparedVisitReport = {
|
||||
totals: { total: 6, valid: 3, invalid: 3, base: 1 },
|
||||
daily: [],
|
||||
weekly: [],
|
||||
monthly: [],
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [],
|
||||
linkCounts: [
|
||||
{ path: '/docs', counts: docs },
|
||||
{ path: '/', counts: root },
|
||||
{ path: '/a%2Fb', counts: empty }
|
||||
]
|
||||
};
|
||||
const period = {
|
||||
timezone: 'UTC' as const,
|
||||
retentionMonths: 3 as const,
|
||||
since: 100,
|
||||
asOf: 200,
|
||||
days: [],
|
||||
weeks: [],
|
||||
months: []
|
||||
};
|
||||
|
||||
test('reads only domain-wide traffic and links to dedicated path routes', async () => {
|
||||
const reports: StoredVisitReports = { period, report, selection: report };
|
||||
vi.mocked(readVisitReport).mockResolvedValue(reports);
|
||||
const data = await load(request());
|
||||
if (!data) throw new Error('Missing page data');
|
||||
expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain);
|
||||
expect(data.totals).toEqual(report.totals);
|
||||
expect(data.invalidClicks).toBe(2);
|
||||
expect(data.linkCount).toBe(3);
|
||||
expect(data.retentionMonths).toBe(3);
|
||||
expect(data.links).toMatchObject([
|
||||
{ href: '/admin/domains/go.example.test/links/docs' },
|
||||
{ href: '/admin/domains/go.example.test/links/a/b' },
|
||||
{ href: '/admin/domains/go.example.test/links/a%252Fb' }
|
||||
]);
|
||||
expect(data.links[2]?.counts).toEqual(empty);
|
||||
expect(data.links[2]?.state).toBe('Disabled');
|
||||
expect(data.base.href).toBe('/admin/domains/go.example.test/links');
|
||||
expect(data.base.url).toBeNull();
|
||||
expect(data.base.state).toBe('Fallback');
|
||||
expect(data.base.counts).toEqual(root);
|
||||
expect(data.asOf).toBe(200);
|
||||
});
|
||||
|
||||
test('shows an active base destination without reclassifying historical outcomes', async () => {
|
||||
vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report });
|
||||
const data = await load(request(domain, 'https://example.test/start'));
|
||||
if (!data) throw new Error('Missing base page data');
|
||||
expect(data.base.url).toBe('https://example.test/start');
|
||||
expect(data.base.state).toBe('Active');
|
||||
expect(data.base.counts).toEqual(root);
|
||||
});
|
||||
|
||||
test('excludes invalid base traffic without subtracting valid base redirects', async () => {
|
||||
const mixedRoot = { total: 5, valid: 3, invalid: 2, base: 5 };
|
||||
const mixedReport: PreparedVisitReport = {
|
||||
...report,
|
||||
totals: { total: 10, valid: 6, invalid: 4, base: 5 },
|
||||
linkCounts: report.linkCounts.map((row) =>
|
||||
row.path === '/' ? { ...row, counts: mixedRoot } : row
|
||||
)
|
||||
};
|
||||
vi.mocked(readVisitReport).mockResolvedValue({
|
||||
period,
|
||||
report: mixedReport,
|
||||
selection: mixedReport
|
||||
});
|
||||
const data = await load(request());
|
||||
if (!data) throw new Error('Missing domain page data');
|
||||
expect(data.invalidClicks).toBe(2);
|
||||
expect(data.totals?.valid).toBe(6);
|
||||
});
|
||||
|
||||
test('rejects unknown domains before reading and preserves unavailable rather than zero', async () => {
|
||||
await expect(load(request('unknown.test'))).rejects.toMatchObject({ status: 404 });
|
||||
expect(readVisitReport).not.toHaveBeenCalled();
|
||||
vi.mocked(readVisitReport).mockResolvedValue(null);
|
||||
const data = await load(request());
|
||||
if (!data) throw new Error('Missing unavailable page');
|
||||
expect(data.available).toBe(false);
|
||||
expect(data.totals).toBeNull();
|
||||
expect(data.invalidClicks).toBeNull();
|
||||
expect(data.links[0]?.counts).toBeNull();
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import type { StoredVisitReports } from '@goto/db/visit-reports';
|
||||
import { beforeEach, expect, test, vi } from 'vitest';
|
||||
import { prepareSnapshot } from '#lib/server/snapshot.js';
|
||||
import { readVisitReport } from '#lib/server/visit-reports.js';
|
||||
import { load } from './+page.server.js';
|
||||
|
||||
vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() }));
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
function request(retention_months = 3) {
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'overview-test-password' } },
|
||||
analytics: { retention_months },
|
||||
domains: {
|
||||
'go.example.test': {
|
||||
links: {
|
||||
'/docs': { url: 'https://example.test/docs' },
|
||||
'/disabled': { url: 'https://example.test/disabled', enabled: false }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
return { locals: { snapshot: prepareSnapshot(prepared) } } as Parameters<typeof load>[0];
|
||||
}
|
||||
|
||||
test('shows current configuration and the worker-prepared snapshot without re-aggregating', async () => {
|
||||
const counts = { total: 4, valid: 2, invalid: 2, base: 1 };
|
||||
const report = {
|
||||
totals: { ...counts, total: 9, valid: 7 },
|
||||
daily: [],
|
||||
weekly: [],
|
||||
monthly: [],
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [{ domain: 'go.example.test', counts }],
|
||||
linkCounts: []
|
||||
};
|
||||
const prepared: StoredVisitReports = {
|
||||
period: {
|
||||
timezone: 'UTC',
|
||||
retentionMonths: 3,
|
||||
since: 100,
|
||||
asOf: 200,
|
||||
days: [],
|
||||
weeks: [],
|
||||
months: []
|
||||
},
|
||||
report,
|
||||
selection: report
|
||||
};
|
||||
vi.mocked(readVisitReport).mockResolvedValue(prepared);
|
||||
const data = await load(request());
|
||||
if (!data) throw new Error('Missing page data');
|
||||
expect(data.domainCount).toBe(1);
|
||||
expect(data.linkCount).toBe(2);
|
||||
expect(data.totals?.total).toBe(9);
|
||||
expect(data.asOf).toBe(200);
|
||||
expect(data.domains[0]?.href).toBe('/admin/domains/go.example.test');
|
||||
});
|
||||
|
||||
test('disabled collection does not read a report; missing preparation is not zero', async () => {
|
||||
const disabled = await load(request(0));
|
||||
if (!disabled) throw new Error('Missing disabled page');
|
||||
expect(disabled.collectionEnabled).toBe(false);
|
||||
expect(disabled.totals).toBeNull();
|
||||
expect(readVisitReport).not.toHaveBeenCalled();
|
||||
vi.mocked(readVisitReport).mockResolvedValue(null);
|
||||
const unavailable = await load(request());
|
||||
if (!unavailable) throw new Error('Missing unavailable page');
|
||||
expect(unavailable.available).toBe(false);
|
||||
expect(unavailable.domains[0]?.counts).toBeNull();
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import type { PreparedVisitReport } from '@goto/db/visit-reports';
|
||||
import { expect, test } from 'vitest';
|
||||
import { buildVisitReports } from './build-visit-reports.js';
|
||||
import { reportPeriod } from './report-buckets.js';
|
||||
|
||||
test('places counts on the correct UTC days, Monday weeks and calendar months', () => {
|
||||
const period = reportPeriod(3, Date.parse('2026-07-01T12:00:00Z'));
|
||||
const reports = buildVisitReports(
|
||||
{
|
||||
daily: [
|
||||
{ scope: 'overview', day: '2026-05-31', total: 3, valid: 2, invalid: 1, base: 1 },
|
||||
{ scope: 'overview', day: '2026-06-01', total: 5, valid: 1, invalid: 4, base: 2 }
|
||||
],
|
||||
browsers: [],
|
||||
operatingSystems: []
|
||||
},
|
||||
[],
|
||||
period
|
||||
);
|
||||
const report = JSON.parse(
|
||||
reports.find((entry) => entry.key === 'overview')!.data
|
||||
) as PreparedVisitReport;
|
||||
const firstDay = period.days.findIndex(
|
||||
(bucket) => bucket.start === Date.parse('2026-05-30T00:00:00Z')
|
||||
);
|
||||
const firstWeek = period.weeks.findIndex(
|
||||
(bucket) => bucket.start === Date.parse('2026-05-25T00:00:00Z')
|
||||
);
|
||||
|
||||
expect(report.daily.slice(firstDay, firstDay + 4)).toEqual([
|
||||
{ total: 0, valid: 0, invalid: 0, base: 0 },
|
||||
{ total: 3, valid: 2, invalid: 1, base: 1 },
|
||||
{ total: 5, valid: 1, invalid: 4, base: 2 },
|
||||
{ total: 0, valid: 0, invalid: 0, base: 0 }
|
||||
]);
|
||||
expect(report.weekly.slice(firstWeek, firstWeek + 2)).toEqual([
|
||||
{ total: 3, valid: 2, invalid: 1, base: 1 },
|
||||
{ total: 5, valid: 1, invalid: 4, base: 2 }
|
||||
]);
|
||||
expect(report.monthly).toEqual([
|
||||
{ total: 0, valid: 0, invalid: 0, base: 0 },
|
||||
{ total: 3, valid: 2, invalid: 1, base: 1 },
|
||||
{ total: 5, valid: 1, invalid: 4, base: 2 },
|
||||
{ total: 0, valid: 0, invalid: 0, base: 0 }
|
||||
]);
|
||||
expect(report.totals).toEqual({ total: 8, valid: 3, invalid: 5, base: 3 });
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import {
|
||||
visitReportKey,
|
||||
zeroVisitCounts,
|
||||
type AggregatedVisitReports,
|
||||
type DailyVisitCounts,
|
||||
type ScopedFamilyCounts,
|
||||
type PreparedVisitReport,
|
||||
type ReportInventory,
|
||||
type SerializedVisitReport,
|
||||
type VisitCounts,
|
||||
type VisitReportPeriod
|
||||
} from '@goto/db/visit-reports';
|
||||
|
||||
interface ScopeCounts {
|
||||
daily: DailyVisitCounts[];
|
||||
browsers: ScopedFamilyCounts[];
|
||||
operatingSystems: ScopedFamilyCounts[];
|
||||
}
|
||||
|
||||
export function buildVisitReports(
|
||||
aggregates: AggregatedVisitReports,
|
||||
inventory: ReportInventory,
|
||||
period: VisitReportPeriod
|
||||
): SerializedVisitReport[] {
|
||||
const indexed = indexCounts(aggregates);
|
||||
const empty = emptyReport(period);
|
||||
const reports: SerializedVisitReport[] = [{ key: 'empty', data: JSON.stringify(empty) }];
|
||||
const linkTotals = new Map<string, VisitCounts>();
|
||||
const domainTotals = new Map<string, VisitCounts>();
|
||||
|
||||
for (const entry of inventory) {
|
||||
const paths = [...new Set([...entry.paths, '/'])];
|
||||
for (const path of paths) {
|
||||
const key = visitReportKey(entry.domain, path);
|
||||
const source = indexed.get(key);
|
||||
if (!source) {
|
||||
linkTotals.set(key, zeroVisitCounts());
|
||||
continue;
|
||||
}
|
||||
const report = scopeReport(source, period);
|
||||
linkTotals.set(key, report.totals);
|
||||
reports.push({ key, data: JSON.stringify(report) });
|
||||
}
|
||||
const key = visitReportKey(entry.domain);
|
||||
const source = indexed.get(key);
|
||||
const report = source ? scopeReport(source, period) : { ...empty };
|
||||
report.linkCounts = paths.map((path) => {
|
||||
const linkKey = visitReportKey(entry.domain, path);
|
||||
return { path, counts: linkTotals.get(linkKey) ?? zeroVisitCounts() };
|
||||
});
|
||||
domainTotals.set(entry.domain, report.totals);
|
||||
reports.push({ key, data: JSON.stringify(report) });
|
||||
}
|
||||
const overviewSource = indexed.get('overview');
|
||||
const overview = overviewSource ? scopeReport(overviewSource, period) : { ...empty };
|
||||
overview.domainCounts = inventory.map((entry) => ({
|
||||
domain: entry.domain,
|
||||
counts: domainTotals.get(entry.domain) ?? zeroVisitCounts()
|
||||
}));
|
||||
reports.push({ key: 'overview', data: JSON.stringify(overview) });
|
||||
return reports;
|
||||
}
|
||||
|
||||
function indexCounts(aggregates: AggregatedVisitReports): Map<string, ScopeCounts> {
|
||||
const indexed = new Map<string, ScopeCounts>();
|
||||
function scope(key: string): ScopeCounts {
|
||||
let counts = indexed.get(key);
|
||||
if (!counts) {
|
||||
counts = { daily: [], browsers: [], operatingSystems: [] };
|
||||
indexed.set(key, counts);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
for (const row of aggregates.daily) scope(row.scope).daily.push(row);
|
||||
for (const row of aggregates.browsers) scope(row.scope).browsers.push(row);
|
||||
for (const row of aggregates.operatingSystems) scope(row.scope).operatingSystems.push(row);
|
||||
return indexed;
|
||||
}
|
||||
|
||||
function emptyReport(period: VisitReportPeriod): PreparedVisitReport {
|
||||
return {
|
||||
totals: zeroVisitCounts(),
|
||||
daily: period.days.map(() => zeroVisitCounts()),
|
||||
weekly: period.weeks.map(() => zeroVisitCounts()),
|
||||
monthly: period.months.map(() => zeroVisitCounts()),
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [],
|
||||
linkCounts: []
|
||||
};
|
||||
}
|
||||
|
||||
function scopeReport(source: ScopeCounts, period: VisitReportPeriod): PreparedVisitReport {
|
||||
const report = emptyReport(period);
|
||||
const dayIndexes = new Map<number, number>();
|
||||
period.days.forEach((bucket, index) => dayIndexes.set(bucket.start, index));
|
||||
for (const row of source.daily) {
|
||||
const start = Date.parse(`${row.day}T00:00:00Z`);
|
||||
const index = dayIndexes.get(start);
|
||||
if (index === undefined) continue;
|
||||
addCounts(report.daily[index]!, row);
|
||||
addCounts(report.totals, row);
|
||||
}
|
||||
for (let index = 0; index < period.days.length; index++) {
|
||||
const day = period.days[index]!;
|
||||
const counts = report.daily[index]!;
|
||||
const week = period.weeks.findIndex(
|
||||
(bucket) => day.start >= bucket.start && day.start < bucket.end
|
||||
);
|
||||
const month = period.months.findIndex(
|
||||
(bucket) => day.start >= bucket.start && day.start < bucket.end
|
||||
);
|
||||
if (week >= 0) addCounts(report.weekly[week]!, counts);
|
||||
if (month >= 0) addCounts(report.monthly[month]!, counts);
|
||||
}
|
||||
report.browsers = source.browsers.map((row) => ({ name: row.name, counts: copyCounts(row) }));
|
||||
report.operatingSystems = source.operatingSystems.map((row) => ({
|
||||
name: row.name,
|
||||
counts: copyCounts(row)
|
||||
}));
|
||||
return report;
|
||||
}
|
||||
|
||||
function copyCounts(counts: VisitCounts): VisitCounts {
|
||||
return { total: counts.total, valid: counts.valid, invalid: counts.invalid, base: counts.base };
|
||||
}
|
||||
|
||||
function addCounts(target: VisitCounts, counts: VisitCounts): void {
|
||||
target.total += counts.total;
|
||||
target.valid += counts.valid;
|
||||
target.invalid += counts.invalid;
|
||||
target.base += counts.base;
|
||||
}
|
||||
@@ -5,10 +5,10 @@ import { applyConfig, watchConfiguration } from './config-reload.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import type { ShutdownResources } from './lifecycle.js';
|
||||
import { logger } from './logger.js';
|
||||
import { requestRetention } from './retention.js';
|
||||
import { requestReportRefresh } from './report-schedule.js';
|
||||
|
||||
vi.mock('./logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }));
|
||||
vi.mock('./retention.js', () => ({ requestRetention: vi.fn() }));
|
||||
vi.mock('./report-schedule.js', () => ({ requestReportRefresh: vi.fn() }));
|
||||
vi.mock('@goto/config', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('@goto/config')>()),
|
||||
startConfigReloading: vi.fn()
|
||||
@@ -29,14 +29,11 @@ function configuration(
|
||||
{ env: {} }
|
||||
);
|
||||
}
|
||||
|
||||
const engine = {} as OpenWorkflow;
|
||||
function context(): WorkerContext {
|
||||
return { config: configuration().configuration, db: {} as WorkerContext['db'] };
|
||||
return { config: configuration().configuration, domains: {}, db: {} as WorkerContext['db'] };
|
||||
}
|
||||
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
@@ -57,7 +54,7 @@ test('requests restart for either database path when collection remains enabled'
|
||||
).toBe(false);
|
||||
expect(restart).toHaveBeenCalledTimes(2);
|
||||
expect(ctx.config).toBe(accepted);
|
||||
expect(requestRetention).not.toHaveBeenCalled();
|
||||
expect(requestReportRefresh).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test.each([
|
||||
@@ -74,43 +71,57 @@ test.each([
|
||||
expect(await applyConfig(next, initial, restart, ctx, engine)).toBe(false);
|
||||
expect(ctx.config).toBe(initial.configuration);
|
||||
expect(restart).not.toHaveBeenCalled();
|
||||
expect(requestRetention).not.toHaveBeenCalled();
|
||||
expect(requestReportRefresh).not.toHaveBeenCalled();
|
||||
expect(logger.warn).toHaveBeenCalledExactlyOnceWith(
|
||||
'Analytics collection change requires coordinated restart; keeping current configuration'
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
test('waits for a positive policy change to be pruned and accepts a later positive change', async () => {
|
||||
test('waits for refreshed reports when retention changes', async () => {
|
||||
const initial = configuration(undefined, 9);
|
||||
const ctx = context();
|
||||
ctx.config = initial.configuration;
|
||||
const pruned = Promise.withResolvers<void>();
|
||||
vi.mocked(requestRetention).mockReturnValueOnce(pruned.promise);
|
||||
const refreshed = Promise.withResolvers<void>();
|
||||
vi.mocked(requestReportRefresh).mockReturnValueOnce(refreshed.promise);
|
||||
const next = configuration();
|
||||
const applying = applyConfig(next, initial, vi.fn(), ctx, engine);
|
||||
await Promise.resolve();
|
||||
expect(ctx.config.analytics.retention_months).toBe(3);
|
||||
expect(requestRetention).toHaveBeenCalledExactlyOnceWith(ctx, engine);
|
||||
expect(requestReportRefresh).toHaveBeenCalledExactlyOnceWith(ctx, engine);
|
||||
let accepted = false;
|
||||
void applying.then(() => {
|
||||
accepted = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(accepted).toBe(false);
|
||||
pruned.resolve();
|
||||
refreshed.resolve();
|
||||
expect(await applying).toBe(true);
|
||||
expect(await applyConfig(configuration(undefined, 12), next, vi.fn(), ctx, engine)).toBe(true);
|
||||
expect(ctx.config.analytics.retention_months).toBe(12);
|
||||
expect(requestRetention).toHaveBeenCalledTimes(2);
|
||||
expect(requestReportRefresh).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
test('a failed positive policy reconciliation requests shutdown', async () => {
|
||||
test('refreshes report inventory after an accepted domain/link change', async () => {
|
||||
const initial = configuration();
|
||||
const next = configuration();
|
||||
next.domains['go.example.test'] = {
|
||||
links: {},
|
||||
fallback: 404,
|
||||
provenance: { fallback: 'builtin' }
|
||||
};
|
||||
const ctx = context();
|
||||
expect(await applyConfig(next, initial, vi.fn(), ctx, engine)).toBe(true);
|
||||
expect(ctx.domains).toBe(next.domains);
|
||||
expect(requestReportRefresh).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test('a failed refresh on positive policy reconciliation requests shutdown', async () => {
|
||||
const initial = configuration(undefined, 9);
|
||||
const ctx = context();
|
||||
ctx.config = initial.configuration;
|
||||
const failure = new Error('Private storage detail');
|
||||
vi.mocked(requestRetention).mockRejectedValueOnce(failure);
|
||||
vi.mocked(requestReportRefresh).mockRejectedValueOnce(failure);
|
||||
const stop = vi.fn();
|
||||
await expect(applyConfig(configuration(), initial, stop, ctx, engine)).rejects.toBe(failure);
|
||||
expect(ctx.config.analytics.retention_months).toBe(3);
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { startConfigReloading, type PreparedConfiguration } from '@goto/config';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import { logger } from './logger.js';
|
||||
import type { ShutdownResources } from './lifecycle.js';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import { requestRetention } from './retention.js';
|
||||
import { requestReportRefresh } from './report-schedule.js';
|
||||
|
||||
interface ConfigurationSource {
|
||||
path: string;
|
||||
@@ -40,9 +41,11 @@ export async function applyConfig(
|
||||
}
|
||||
const currentMonths = context.config.analytics.retention_months;
|
||||
context.config = next.configuration;
|
||||
context.domains = next.domains;
|
||||
try {
|
||||
if (months !== previousMonths || months !== currentMonths) {
|
||||
await requestRetention(context, engine);
|
||||
const inventoryChanged = !isDeepStrictEqual(next.domains, previous.domains);
|
||||
if (months !== previousMonths || months !== currentMonths || inventoryChanged) {
|
||||
await requestReportRefresh(context, engine);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Unable to apply worker retention policy; stopping service');
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import type { GotoConfiguration } from '@goto/config';
|
||||
import type { GotoConfiguration, PreparedConfiguration } from '@goto/config';
|
||||
import type { openDb } from '@goto/db';
|
||||
|
||||
export interface WorkerContext {
|
||||
config: GotoConfiguration;
|
||||
domains: PreparedConfiguration['domains'];
|
||||
db: Awaited<ReturnType<typeof openDb>>;
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import { openDb, runMigrations, insertVisit, pruneVisits, type ClassifiedVisit } from '@goto/db';
|
||||
import { readPreparedVisitReports } from '@goto/db/visit-reports';
|
||||
import { afterEach, expect, test, vi } from 'vitest';
|
||||
import { prepareVisitReports } from './prepare-visit-reports.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
|
||||
vi.mock('@goto/db', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import('@goto/db')>();
|
||||
return { ...actual, pruneVisits: vi.fn(actual.pruneVisits) };
|
||||
});
|
||||
|
||||
afterEach(() => vi.mocked(pruneVisits).mockReset());
|
||||
|
||||
async function context(): Promise<WorkerContext> {
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'prepare-test-password' } },
|
||||
domains: { 'go.example.test': { links: { '/docs': { url: 'https://example.test/docs' } } } }
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
const db = await openDb(':memory:');
|
||||
await runMigrations(db);
|
||||
return { config: prepared.configuration, domains: prepared.domains, db };
|
||||
}
|
||||
const visit: ClassifiedVisit = {
|
||||
domain: 'go.example.test',
|
||||
path: '/docs',
|
||||
outcome: 'valid' as const,
|
||||
visited_at: 1,
|
||||
ip: '127.0.0.1',
|
||||
user_agent: null,
|
||||
accept_language: null,
|
||||
browser: 'Firefox',
|
||||
os: 'Linux'
|
||||
};
|
||||
|
||||
function sum(values: { total: number }[]): number {
|
||||
return values.reduce((total, value) => total + value.total, 0);
|
||||
}
|
||||
|
||||
test('one refresh publishes all scopes and chart resolutions, excludes/deletes expired data and is repeatable', async () => {
|
||||
const ctx = await context();
|
||||
try {
|
||||
const now = Date.parse('2026-05-31T12:00:00Z');
|
||||
const cutoff = Date.parse('2026-02-28T12:00:00Z');
|
||||
await ctx.db
|
||||
.insertInto('visits')
|
||||
.values(Array.from({ length: 1001 }, () => ({ ...visit, visited_at: cutoff - 1 })))
|
||||
.execute();
|
||||
await insertVisit(ctx.db, { ...visit, visited_at: cutoff });
|
||||
await insertVisit(ctx.db, { ...visit, path: '/', outcome: 'invalid', visited_at: now - 2000 });
|
||||
await prepareVisitReports(ctx, now);
|
||||
const reports = await readPreparedVisitReports(ctx.db, 'go.example.test', '/docs');
|
||||
expect(reports?.report.totals).toEqual({ total: 2, valid: 1, invalid: 1, base: 1 });
|
||||
expect(reports?.selection.totals.total).toBe(1);
|
||||
expect(sum(reports!.report.daily)).toBe(2);
|
||||
expect(sum(reports!.report.weekly)).toBe(2);
|
||||
expect(sum(reports!.report.monthly)).toBe(2);
|
||||
expect(reports?.report.browsers).toEqual([
|
||||
{ name: 'Firefox', counts: { total: 2, valid: 1, invalid: 1, base: 1 } }
|
||||
]);
|
||||
expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(2);
|
||||
await prepareVisitReports(ctx, now);
|
||||
expect((await readPreparedVisitReports(ctx.db))?.report.totals.total).toBe(2);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('zero purges raw history and prepared output without leaving a longer-lived total', async () => {
|
||||
const ctx = await context();
|
||||
try {
|
||||
const now = Date.now();
|
||||
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
|
||||
await prepareVisitReports(ctx, now);
|
||||
ctx.config.analytics.retention_months = 0;
|
||||
await prepareVisitReports(ctx, now);
|
||||
expect(await ctx.db.selectFrom('visits').select('id').execute()).toEqual([]);
|
||||
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('clears old output before pruning and publishes nothing if pruning fails', async () => {
|
||||
const ctx = await context();
|
||||
try {
|
||||
const now = Date.parse('2026-05-31T12:00:00Z');
|
||||
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
|
||||
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
|
||||
const failure = new Error('Pruning failed');
|
||||
vi.mocked(pruneVisits).mockImplementationOnce(async () => {
|
||||
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
|
||||
throw failure;
|
||||
});
|
||||
|
||||
await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure);
|
||||
expect(pruneVisits).toHaveBeenCalledExactlyOnceWith(ctx.db, Date.parse('2026-02-28T12:00:00Z'));
|
||||
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('stops pruning and does not publish obsolete output when positive retention changes', async () => {
|
||||
const ctx = await context();
|
||||
try {
|
||||
const now = Date.parse('2026-05-31T12:00:00Z');
|
||||
await insertVisit(ctx.db, { ...visit, visited_at: now - 1 });
|
||||
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
|
||||
vi.mocked(pruneVisits).mockImplementationOnce(async () => {
|
||||
ctx.config.analytics.retention_months = 9;
|
||||
return true;
|
||||
});
|
||||
|
||||
await prepareVisitReports(ctx, now);
|
||||
expect(pruneVisits).toHaveBeenCalledOnce();
|
||||
expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import { pruneVisits, purgeVisits } from '@goto/db';
|
||||
import {
|
||||
aggregateVisitReports,
|
||||
clearVisitReports,
|
||||
publishVisitReports,
|
||||
type ReportInventory
|
||||
} from '@goto/db/visit-reports';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import { reportPeriod } from './report-buckets.js';
|
||||
import { buildVisitReports } from './build-visit-reports.js';
|
||||
|
||||
export async function prepareVisitReports(
|
||||
context: WorkerContext,
|
||||
asOf = Date.now()
|
||||
): Promise<void> {
|
||||
const months = context.config.analytics.retention_months;
|
||||
if (months === 0) {
|
||||
await context.db.transaction().execute(async (transaction) => {
|
||||
await clearVisitReports(transaction);
|
||||
await purgeVisits(transaction);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const inventory: ReportInventory = Object.entries(context.domains).map(([domain, prepared]) => ({
|
||||
domain,
|
||||
paths: Object.keys(prepared.links)
|
||||
}));
|
||||
const period = reportPeriod(months, asOf);
|
||||
const counts = await aggregateVisitReports(context.db, period.since, asOf, inventory);
|
||||
const reports = buildVisitReports(counts, inventory, period);
|
||||
if (context.config.analytics.retention_months !== months) return;
|
||||
|
||||
// Retire old output before deleting any history represented by it.
|
||||
await clearVisitReports(context.db);
|
||||
while (context.config.analytics.retention_months === months) {
|
||||
const more = await pruneVisits(context.db, period.since);
|
||||
if (!more) break;
|
||||
await setImmediate();
|
||||
}
|
||||
if (context.config.analytics.retention_months !== months) return;
|
||||
await publishVisitReports(context.db, period, reports);
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import { expect, test } from 'vitest';
|
||||
import { reportPeriod } from './report-buckets.js';
|
||||
|
||||
test('prepares UTC days, Monday weeks and calendar months with honest partial edges', () => {
|
||||
const asOf = Date.parse('2026-05-31T12:34:56.789Z');
|
||||
const period = reportPeriod(3, asOf);
|
||||
expect(period.since).toBe(Date.parse('2026-02-28T12:34:56.789Z'));
|
||||
expect(period.asOf).toBe(asOf);
|
||||
expect(period.days[0]).toEqual({
|
||||
start: Date.parse('2026-02-28T00:00:00Z'),
|
||||
end: Date.parse('2026-03-01T00:00:00Z'),
|
||||
partial: true
|
||||
});
|
||||
expect(period.days[1]).toEqual({
|
||||
start: Date.parse('2026-03-01T00:00:00Z'),
|
||||
end: Date.parse('2026-03-02T00:00:00Z'),
|
||||
partial: false
|
||||
});
|
||||
expect(period.weeks[1]).toEqual({
|
||||
start: Date.parse('2026-03-02T00:00:00Z'),
|
||||
end: Date.parse('2026-03-09T00:00:00Z'),
|
||||
partial: false
|
||||
});
|
||||
expect(period.months[1]).toEqual({
|
||||
start: Date.parse('2026-03-01T00:00:00Z'),
|
||||
end: Date.parse('2026-04-01T00:00:00Z'),
|
||||
partial: false
|
||||
});
|
||||
expect(new Date(period.weeks[0]!.start).getUTCDay()).toBe(1);
|
||||
expect(period.months[0]!.start).toBe(Date.parse('2026-02-01T00:00:00Z'));
|
||||
expect(period.days.at(-1)!.partial).toBe(true);
|
||||
expect(period.months.at(-1)!.partial).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { TimeBucket, VisitReportPeriod } from '@goto/db/visit-reports';
|
||||
import { retentionCutoff, type RetentionMonths } from './retention.js';
|
||||
|
||||
type BucketSize = 'day' | 'week' | 'month';
|
||||
|
||||
export function reportPeriod(months: Exclude<RetentionMonths, 0>, asOf: number): VisitReportPeriod {
|
||||
const since = retentionCutoff(months, asOf);
|
||||
return {
|
||||
timezone: 'UTC',
|
||||
retentionMonths: months,
|
||||
since,
|
||||
asOf,
|
||||
days: timeBuckets(since, asOf, 'day'),
|
||||
weeks: timeBuckets(since, asOf, 'week'),
|
||||
months: timeBuckets(since, asOf, 'month')
|
||||
};
|
||||
}
|
||||
|
||||
function timeBuckets(since: number, asOf: number, size: BucketSize): TimeBucket[] {
|
||||
const date = new Date(since);
|
||||
date.setUTCHours(0, 0, 0, 0);
|
||||
if (size === 'week') {
|
||||
const daysSinceMonday = (date.getUTCDay() + 6) % 7;
|
||||
date.setUTCDate(date.getUTCDate() - daysSinceMonday);
|
||||
}
|
||||
if (size === 'month') date.setUTCDate(1);
|
||||
const buckets: TimeBucket[] = [];
|
||||
while (date.getTime() < asOf) {
|
||||
const start = date.getTime();
|
||||
if (size === 'month') date.setUTCMonth(date.getUTCMonth() + 1);
|
||||
else date.setUTCDate(date.getUTCDate() + (size === 'week' ? 7 : 1));
|
||||
const end = date.getTime();
|
||||
buckets.push({ start, end, partial: start < since || end > asOf });
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import { openDb, runMigrations } from '@goto/db';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import { expect, test, vi } from 'vitest';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import { configureReportSchedule, requestReportRefresh } from './report-schedule.js';
|
||||
import { refreshVisitReports, reportSchedule } from '../workflows/specs.js';
|
||||
|
||||
async function context(): Promise<WorkerContext> {
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'schedule-test-password' } },
|
||||
domains: {}
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
const db = await openDb(':memory:');
|
||||
await runMigrations(db);
|
||||
return { config: prepared.configuration, domains: prepared.domains, db };
|
||||
}
|
||||
function client() {
|
||||
const result = vi.fn(async () => 'completed');
|
||||
const engine = {
|
||||
scheduleWorkflow: vi.fn(async () => {}),
|
||||
schedules: { pause: vi.fn(async () => {}), resume: vi.fn(async () => {}) },
|
||||
runWorkflow: vi.fn(async () => ({ result }))
|
||||
};
|
||||
return { engine, sdk: engine as unknown as OpenWorkflow, result };
|
||||
}
|
||||
|
||||
test('zero pauses hourly scheduling and purges prepared output even if raw history is already empty', async () => {
|
||||
const ctx = await context();
|
||||
const { engine, sdk } = client();
|
||||
try {
|
||||
ctx.config.analytics.retention_months = 0;
|
||||
await configureReportSchedule(ctx, sdk);
|
||||
expect(engine.scheduleWorkflow).toHaveBeenCalledExactlyOnceWith(
|
||||
refreshVisitReports,
|
||||
reportSchedule.input,
|
||||
reportSchedule
|
||||
);
|
||||
expect(refreshVisitReports.name).toBe('retain-visits');
|
||||
expect(reportSchedule.id).toBe('goto:retain-visits');
|
||||
expect(reportSchedule.cron).toBe('0 * * * *');
|
||||
expect(reportSchedule.timezone).toBe('UTC');
|
||||
expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(reportSchedule.id);
|
||||
await requestReportRefresh(ctx, sdk);
|
||||
expect(engine.runWorkflow).not.toHaveBeenCalled();
|
||||
await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute();
|
||||
await requestReportRefresh(ctx, sdk);
|
||||
expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(refreshVisitReports, {});
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('positive policy resumes scheduling and startup waits for the preparation result', async () => {
|
||||
const ctx = await context();
|
||||
const { engine, sdk, result } = client();
|
||||
try {
|
||||
await configureReportSchedule(ctx, sdk);
|
||||
expect(engine.schedules.resume).toHaveBeenCalledExactlyOnceWith(reportSchedule.id);
|
||||
const completed = Promise.withResolvers<string>();
|
||||
const requested = Promise.withResolvers<void>();
|
||||
result.mockImplementationOnce(() => {
|
||||
requested.resolve();
|
||||
return completed.promise;
|
||||
});
|
||||
let finished = false;
|
||||
const preparing = requestReportRefresh(ctx, sdk).then(() => {
|
||||
finished = true;
|
||||
});
|
||||
await requested.promise;
|
||||
expect(finished).toBe(false);
|
||||
completed.resolve('completed');
|
||||
await preparing;
|
||||
expect(finished).toBe(true);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import { refreshVisitReports, reportSchedule } from '../workflows/specs.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
|
||||
export async function configureReportSchedule(
|
||||
context: WorkerContext,
|
||||
engine: OpenWorkflow
|
||||
): Promise<void> {
|
||||
await engine.scheduleWorkflow(refreshVisitReports, reportSchedule.input, reportSchedule);
|
||||
if (context.config.analytics.retention_months === 0)
|
||||
await engine.schedules.pause(reportSchedule.id);
|
||||
else await engine.schedules.resume(reportSchedule.id);
|
||||
}
|
||||
|
||||
export async function requestReportRefresh(
|
||||
context: WorkerContext,
|
||||
engine: OpenWorkflow
|
||||
): Promise<void> {
|
||||
if (context.config.analytics.retention_months === 0) {
|
||||
const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst();
|
||||
const output = await context.db
|
||||
.selectFrom('visit_reports')
|
||||
.select('key')
|
||||
.limit(1)
|
||||
.executeTakeFirst();
|
||||
if (!visit && !output) return;
|
||||
}
|
||||
const run = await engine.runWorkflow(refreshVisitReports, {});
|
||||
await run.result();
|
||||
}
|
||||
@@ -1,15 +1,5 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import { openDb, runMigrations, type ClassifiedVisit } from '@goto/db';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import { expect, test, vi } from 'vitest';
|
||||
import {
|
||||
applyRetention,
|
||||
configureRetentionSchedule,
|
||||
requestRetention,
|
||||
retentionCutoff
|
||||
} from './retention.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import { retainVisits, retentionSchedule } from '../workflows/specs.js';
|
||||
import { expect, test } from 'vitest';
|
||||
import { retentionCutoff } from './retention.js';
|
||||
|
||||
test('uses UTC calendar months, clamping the day while preserving the time', () => {
|
||||
const now = Date.parse('2026-05-31T12:34:56.789Z');
|
||||
@@ -18,107 +8,6 @@ test('uses UTC calendar months, clamping the day while preserving the time', ()
|
||||
expect(new Date(retentionCutoff(12, now)).toISOString()).toBe('2025-05-31T12:34:56.789Z');
|
||||
});
|
||||
|
||||
async function context(): Promise<WorkerContext> {
|
||||
const db = await openDb(':memory:');
|
||||
await runMigrations(db);
|
||||
const { configuration: config } = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'retention-test-password' } },
|
||||
domains: {}
|
||||
},
|
||||
{ env: {} }
|
||||
);
|
||||
return { db, config };
|
||||
}
|
||||
|
||||
function client() {
|
||||
const result = vi.fn(async () => 'completed');
|
||||
const engine = {
|
||||
scheduleWorkflow: vi.fn(async () => {}),
|
||||
schedules: { pause: vi.fn(async () => {}), resume: vi.fn(async () => {}) },
|
||||
runWorkflow: vi.fn(async () => ({ result }))
|
||||
};
|
||||
return { engine, result };
|
||||
}
|
||||
|
||||
const visit: ClassifiedVisit = {
|
||||
domain: 'go.example.test',
|
||||
path: '/docs',
|
||||
outcome: 'valid',
|
||||
visited_at: 1,
|
||||
ip: '127.0.0.1',
|
||||
user_agent: null,
|
||||
accept_language: null,
|
||||
browser: 'Other',
|
||||
os: 'Other'
|
||||
};
|
||||
|
||||
test('zero pauses the native schedule and queues one purge only when data exists', async () => {
|
||||
const ctx = await context();
|
||||
const { engine, result } = client();
|
||||
const sdk = engine as unknown as OpenWorkflow;
|
||||
try {
|
||||
ctx.config.analytics.retention_months = 0;
|
||||
await configureRetentionSchedule(ctx, sdk);
|
||||
expect(engine.scheduleWorkflow).toHaveBeenCalledExactlyOnceWith(
|
||||
retainVisits,
|
||||
retentionSchedule.input,
|
||||
retentionSchedule
|
||||
);
|
||||
expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(retentionSchedule.id);
|
||||
expect(engine.schedules.resume).not.toHaveBeenCalled();
|
||||
await requestRetention(ctx, sdk);
|
||||
expect(engine.runWorkflow).not.toHaveBeenCalled();
|
||||
await ctx.db.insertInto('visits').values(visit).execute();
|
||||
const completed = Promise.withResolvers<string>();
|
||||
const requested = Promise.withResolvers<void>();
|
||||
result.mockImplementationOnce(() => {
|
||||
requested.resolve();
|
||||
return completed.promise;
|
||||
});
|
||||
const requesting = requestRetention(ctx, sdk);
|
||||
await requested.promise;
|
||||
let finished = false;
|
||||
void requesting.then(() => {
|
||||
finished = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(finished).toBe(false);
|
||||
completed.resolve('completed');
|
||||
await requesting;
|
||||
expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(retainVisits, {});
|
||||
await applyRetention(ctx);
|
||||
expect(await ctx.db.selectFrom('visits').selectAll().execute()).toEqual([]);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('positive retention resumes the schedule and prunes every expired batch using one cutoff', async () => {
|
||||
const ctx = await context();
|
||||
const { engine } = client();
|
||||
try {
|
||||
const now = Date.parse('2026-05-31T12:34:56.789Z');
|
||||
const cutoff = retentionCutoff(3, now);
|
||||
await ctx.db
|
||||
.insertInto('visits')
|
||||
.values([
|
||||
...Array.from({ length: 1001 }, () => ({ ...visit, visited_at: cutoff - 1 })),
|
||||
{ ...visit, visited_at: cutoff }
|
||||
])
|
||||
.execute();
|
||||
await configureRetentionSchedule(ctx, engine as unknown as OpenWorkflow);
|
||||
expect(engine.schedules.resume).toHaveBeenCalledExactlyOnceWith(retentionSchedule.id);
|
||||
expect(engine.schedules.pause).not.toHaveBeenCalled();
|
||||
await applyRetention(ctx, now);
|
||||
expect(await ctx.db.selectFrom('visits').select('visited_at').execute()).toEqual([
|
||||
{ visited_at: cutoff }
|
||||
]);
|
||||
} finally {
|
||||
await ctx.db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
test('handles leap-day boundaries without overflowing into March', () => {
|
||||
expect(new Date(retentionCutoff(3, Date.parse('2024-05-31T01:00:00Z'))).toISOString()).toBe(
|
||||
'2024-02-29T01:00:00.000Z'
|
||||
|
||||
@@ -1,46 +1,7 @@
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import type { GotoConfiguration } from '@goto/config';
|
||||
import { pruneVisits, purgeVisits } from '@goto/db';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import { retainVisits, retentionSchedule } from '../workflows/specs.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
|
||||
export type RetentionMonths = GotoConfiguration['analytics']['retention_months'];
|
||||
|
||||
export async function configureRetentionSchedule(
|
||||
context: WorkerContext,
|
||||
engine: OpenWorkflow
|
||||
): Promise<void> {
|
||||
const enabled = context.config.analytics.retention_months !== 0;
|
||||
await engine.scheduleWorkflow(retainVisits, retentionSchedule.input, retentionSchedule);
|
||||
if (enabled) await engine.schedules.resume(retentionSchedule.id);
|
||||
else await engine.schedules.pause(retentionSchedule.id);
|
||||
}
|
||||
|
||||
export async function requestRetention(
|
||||
context: WorkerContext,
|
||||
engine: OpenWorkflow
|
||||
): Promise<void> {
|
||||
if (context.config.analytics.retention_months === 0) {
|
||||
const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst();
|
||||
if (!visit) return;
|
||||
}
|
||||
const run = await engine.runWorkflow(retainVisits, {});
|
||||
await run.result();
|
||||
}
|
||||
|
||||
export async function applyRetention(context: WorkerContext, now = Date.now()): Promise<void> {
|
||||
const months = context.config.analytics.retention_months;
|
||||
if (months === 0) {
|
||||
await purgeVisits(context.db);
|
||||
return;
|
||||
}
|
||||
const before = retentionCutoff(months, now);
|
||||
while (await pruneVisits(context.db, before)) {
|
||||
await setImmediate();
|
||||
}
|
||||
}
|
||||
|
||||
export function retentionCutoff(months: Exclude<RetentionMonths, 0>, now = Date.now()): number {
|
||||
const cutoff = new Date(now);
|
||||
const day = cutoff.getUTCDate();
|
||||
|
||||
@@ -11,8 +11,8 @@ import { createShutdown, type ShutdownResources } from './lifecycle.js';
|
||||
import { watchConfiguration } from './config-reload.js';
|
||||
import { logger } from './logger.js';
|
||||
import type { WorkerContext } from './context.js';
|
||||
import { configureRetentionSchedule, requestRetention } from './retention.js';
|
||||
import { registerRetentionWorkflow } from '../workflows/retain-visits.js';
|
||||
import { configureReportSchedule, requestReportRefresh } from './report-schedule.js';
|
||||
import { registerReportWorkflow } from '../workflows/refresh-visit-reports.js';
|
||||
|
||||
export interface WorkerService {
|
||||
port: number;
|
||||
@@ -44,7 +44,11 @@ export async function startWorkerService(
|
||||
const stop = createShutdown(resources);
|
||||
try {
|
||||
const db = await openVisitDatabase(visitDatabasePath, resources);
|
||||
const context: WorkerContext = { config: prepared.configuration, db };
|
||||
const context: WorkerContext = {
|
||||
config: prepared.configuration,
|
||||
domains: prepared.domains,
|
||||
db
|
||||
};
|
||||
const engine = await startWorkflowWorker(databasePath, context, resources);
|
||||
const host = captured.env.HOST ?? '0.0.0.0';
|
||||
const healthPort = await startHealthServer(host, port, resources);
|
||||
@@ -79,12 +83,12 @@ async function startWorkflowWorker(
|
||||
resources.backend = backend;
|
||||
const engineLogger = createEngineLogger(logger);
|
||||
const engine = new OpenWorkflow({ backend, logger: engineLogger });
|
||||
registerRetentionWorkflow(engine, context);
|
||||
await configureRetentionSchedule(context, engine);
|
||||
registerReportWorkflow(engine, context);
|
||||
await configureReportSchedule(context, engine);
|
||||
const workflowWorker = engine.newWorker({ concurrency: 1 });
|
||||
resources.workflowWorker = workflowWorker;
|
||||
await workflowWorker.start();
|
||||
await requestRetention(context, engine);
|
||||
await requestReportRefresh(context, engine);
|
||||
logger.info('Workflow worker started');
|
||||
return engine;
|
||||
}
|
||||
|
||||
+20
-20
@@ -1,30 +1,31 @@
|
||||
import { validateConfig } from '@goto/config';
|
||||
import { afterEach, expect, test, vi } from 'vitest';
|
||||
import type { WorkerContext } from '../services/context.js';
|
||||
import { applyRetention } from '../services/retention.js';
|
||||
import { runRetentionWorkflow, type RetentionStep } from './retain-visits.js';
|
||||
|
||||
vi.mock('../services/retention.js', () => ({ applyRetention: vi.fn(async () => {}) }));
|
||||
import { prepareVisitReports } from '../services/prepare-visit-reports.js';
|
||||
import { runReportRefresh, type ReportStep } from './refresh-visit-reports.js';
|
||||
|
||||
vi.mock('../services/prepare-visit-reports.js', () => ({
|
||||
prepareVisitReports: vi.fn(async () => {})
|
||||
}));
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
|
||||
function context(): WorkerContext {
|
||||
const prepared = validateConfig(
|
||||
{
|
||||
auth: { credentials: { username: 'test', password: 'jitter-test-password' } },
|
||||
domains: {}
|
||||
},
|
||||
{ auth: { credentials: { username: 'test', password: 'jitter-test-password' } }, domains: {} },
|
||||
{ env: {} }
|
||||
);
|
||||
return { config: prepared.configuration, db: {} as WorkerContext['db'] };
|
||||
return {
|
||||
config: prepared.configuration,
|
||||
domains: prepared.domains,
|
||||
db: {} as WorkerContext['db']
|
||||
};
|
||||
}
|
||||
|
||||
function steps() {
|
||||
const names: string[] = [];
|
||||
const sleep = vi
|
||||
.fn<(name: string, duration: string) => Promise<void>>()
|
||||
.mockResolvedValue(undefined);
|
||||
const step: RetentionStep = {
|
||||
const step: ReportStep = {
|
||||
async run(config, fn) {
|
||||
names.push(config.name);
|
||||
return fn();
|
||||
@@ -34,33 +35,32 @@ function steps() {
|
||||
return { step, names, sleep };
|
||||
}
|
||||
|
||||
test('scheduled cleanup uses durable jitter; startup and policy runs do not', async () => {
|
||||
test('hourly refresh uses durable jitter; startup/manual refreshes do not', async () => {
|
||||
const ctx = context();
|
||||
const { step, names, sleep } = steps();
|
||||
await runRetentionWorkflow(ctx, { scheduled: true }, step);
|
||||
await runReportRefresh(ctx, { scheduled: true }, step);
|
||||
expect(names).toEqual(['choose-jitter', 'retain-visits']);
|
||||
expect(sleep).toHaveBeenCalledOnce();
|
||||
const [name, duration] = sleep.mock.calls[0]!;
|
||||
const seconds = Number(duration.slice(0, -1));
|
||||
expect(name).toBe('retention-jitter');
|
||||
expect(duration.endsWith('s')).toBe(true);
|
||||
expect(seconds).toBeGreaterThanOrEqual(1);
|
||||
expect(seconds).toBeLessThanOrEqual(60);
|
||||
expect(applyRetention).toHaveBeenCalledExactlyOnceWith(ctx);
|
||||
expect(prepareVisitReports).toHaveBeenCalledExactlyOnceWith(ctx);
|
||||
vi.clearAllMocks();
|
||||
names.length = 0;
|
||||
await runRetentionWorkflow(ctx, {}, step);
|
||||
await runReportRefresh(ctx, {}, step);
|
||||
expect(names).toEqual(['retain-visits']);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
expect(applyRetention).toHaveBeenCalledExactlyOnceWith(ctx);
|
||||
expect(prepareVisitReports).toHaveBeenCalledExactlyOnceWith(ctx);
|
||||
});
|
||||
|
||||
test('a prior scheduled run does no cleanup when the service starts with collection disabled', async () => {
|
||||
test('old scheduled work does nothing when the service starts disabled', async () => {
|
||||
const ctx = context();
|
||||
ctx.config.analytics.retention_months = 0;
|
||||
const { step, names, sleep } = steps();
|
||||
await runRetentionWorkflow(ctx, { scheduled: true }, step);
|
||||
await runReportRefresh(ctx, { scheduled: true }, step);
|
||||
expect(names).toEqual([]);
|
||||
expect(sleep).not.toHaveBeenCalled();
|
||||
expect(applyRetention).not.toHaveBeenCalled();
|
||||
expect(prepareVisitReports).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { randomInt } from 'node:crypto';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import type { WorkerContext } from '../services/context.js';
|
||||
import { prepareVisitReports } from '../services/prepare-visit-reports.js';
|
||||
import { refreshVisitReports, type ReportRefreshInput } from './specs.js';
|
||||
|
||||
const MAX_JITTER_SECONDS = 60;
|
||||
|
||||
export interface ReportStep {
|
||||
run<T>(config: { name: string }, fn: () => Promise<T>): Promise<T>;
|
||||
sleep(name: string, duration: `${number}s`): Promise<void>;
|
||||
}
|
||||
|
||||
export async function runReportRefresh(
|
||||
context: WorkerContext,
|
||||
input: ReportRefreshInput,
|
||||
step: ReportStep
|
||||
): Promise<'completed'> {
|
||||
if (input.scheduled) {
|
||||
if (context.config.analytics.retention_months === 0) return 'completed';
|
||||
const seconds = await step.run({ name: 'choose-jitter' }, chooseJitter);
|
||||
const duration: `${number}s` = `${seconds}s`;
|
||||
await step.sleep('retention-jitter', duration);
|
||||
}
|
||||
function prepare(): Promise<void> {
|
||||
return prepareVisitReports(context);
|
||||
}
|
||||
// Preserve the durable step identity used by already-queued maintenance runs.
|
||||
await step.run({ name: 'retain-visits' }, prepare);
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
export function registerReportWorkflow(engine: OpenWorkflow, context: WorkerContext): void {
|
||||
function execute({ input, step }: { input: ReportRefreshInput; step: ReportStep }) {
|
||||
return runReportRefresh(context, input, step);
|
||||
}
|
||||
engine.implementWorkflow(refreshVisitReports, execute);
|
||||
}
|
||||
|
||||
async function chooseJitter(): Promise<number> {
|
||||
const upperBound = MAX_JITTER_SECONDS + 1;
|
||||
return randomInt(1, upperBound);
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import { randomInt } from 'node:crypto';
|
||||
import type { OpenWorkflow } from 'openworkflow';
|
||||
import type { WorkerContext } from '../services/context.js';
|
||||
import { applyRetention } from '../services/retention.js';
|
||||
import { retainVisits, type RetainVisitsInput } from './specs.js';
|
||||
|
||||
const MAX_JITTER_SECONDS = 60;
|
||||
|
||||
export interface RetentionStep {
|
||||
run<T>(config: { name: string }, fn: () => Promise<T>): Promise<T>;
|
||||
sleep(name: string, duration: `${number}s`): Promise<void>;
|
||||
}
|
||||
|
||||
function collectionDisabled(context: WorkerContext): boolean {
|
||||
return context.config.analytics.retention_months === 0;
|
||||
}
|
||||
|
||||
async function chooseJitter(): Promise<number> {
|
||||
const upperBound = MAX_JITTER_SECONDS + 1;
|
||||
return randomInt(1, upperBound);
|
||||
}
|
||||
|
||||
export async function runRetentionWorkflow(
|
||||
context: WorkerContext,
|
||||
input: RetainVisitsInput,
|
||||
step: RetentionStep
|
||||
): Promise<'completed'> {
|
||||
if (input.scheduled) {
|
||||
if (collectionDisabled(context)) return 'completed';
|
||||
const seconds = await step.run({ name: 'choose-jitter' }, chooseJitter);
|
||||
const duration: `${number}s` = `${seconds}s`;
|
||||
await step.sleep('retention-jitter', duration);
|
||||
}
|
||||
function applyPolicy(): Promise<void> {
|
||||
return applyRetention(context);
|
||||
}
|
||||
await step.run({ name: 'retain-visits' }, applyPolicy);
|
||||
return 'completed';
|
||||
}
|
||||
|
||||
export function registerRetentionWorkflow(engine: OpenWorkflow, context: WorkerContext): void {
|
||||
function execute({ input, step }: { input: RetainVisitsInput; step: RetentionStep }) {
|
||||
return runRetentionWorkflow(context, input, step);
|
||||
}
|
||||
engine.implementWorkflow(retainVisits, execute);
|
||||
}
|
||||
@@ -1,18 +1,22 @@
|
||||
import { defineWorkflowSpec, type Workflow } from 'openworkflow';
|
||||
|
||||
export interface RetainVisitsInput {
|
||||
export interface ReportRefreshInput {
|
||||
scheduled?: boolean;
|
||||
}
|
||||
|
||||
export const retentionSchedule = {
|
||||
export const reportSchedule = {
|
||||
id: 'goto:retain-visits',
|
||||
cron: '0 * * * *',
|
||||
timezone: 'UTC',
|
||||
input: { scheduled: true }
|
||||
};
|
||||
|
||||
export const retainVisits: Workflow<RetainVisitsInput, 'completed', RetainVisitsInput>['spec'] =
|
||||
defineWorkflowSpec<RetainVisitsInput, 'completed'>({
|
||||
name: 'retain-visits',
|
||||
schedules: [retentionSchedule]
|
||||
});
|
||||
// Keep the durable identity so existing schedules and queued runs remain usable.
|
||||
export const refreshVisitReports: Workflow<
|
||||
ReportRefreshInput,
|
||||
'completed',
|
||||
ReportRefreshInput
|
||||
>['spec'] = defineWorkflowSpec<ReportRefreshInput, 'completed'>({
|
||||
name: 'retain-visits',
|
||||
schedules: [reportSchedule]
|
||||
});
|
||||
+4
-2
@@ -11,6 +11,7 @@
|
||||
"dev": "pnpm build:packages && pnpm --filter @goto/web dev",
|
||||
"dev:worker": "pnpm --filter @goto/worker dev",
|
||||
"cli": "pnpm build:packages && pnpm --filter @goto/cli build && node apps/cli/dist/index.js",
|
||||
"seed:visits": "node --import ./tools/register.mjs scripts/seed-visits.js",
|
||||
"schema:generate": "pnpm --filter @goto/config schema:generate",
|
||||
"schema:check": "pnpm --filter @goto/config schema:check",
|
||||
"build:packages": "pnpm --filter './packages/*' -r build",
|
||||
@@ -18,8 +19,9 @@
|
||||
"check": "pnpm -r check",
|
||||
"lint": "prettier --check . && eslint .",
|
||||
"format": "prettier --write .",
|
||||
"test": "pnpm -r test",
|
||||
"test:coverage": "pnpm -r test:coverage",
|
||||
"test:scripts": "node --import ./tools/register.mjs --test scripts/*.test.js",
|
||||
"test": "pnpm test:scripts && pnpm -r test",
|
||||
"test:coverage": "pnpm test:scripts && pnpm -r test:coverage",
|
||||
"test:e2e": "pnpm --filter @goto/web test:e2e",
|
||||
"verify": "pnpm schema:check && pnpm build && pnpm check && pnpm lint && pnpm test:coverage && pnpm test:e2e && pnpm --filter @goto/worker test:smoke",
|
||||
"changeset": "changeset",
|
||||
|
||||
@@ -9,7 +9,12 @@
|
||||
"development": "./src/index.ts",
|
||||
"import": "./dist/index.js"
|
||||
},
|
||||
"./visit-worker": "./dist/visit-worker.js"
|
||||
"./visit-worker": "./dist/visit-worker.js",
|
||||
"./visit-reports": {
|
||||
"development": "./src/visit-reports/index.ts",
|
||||
"types": "./dist/visit-reports/index.d.ts",
|
||||
"import": "./dist/visit-reports/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist"
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { Kysely, Migration } from 'kysely';
|
||||
|
||||
export const visitReports: Migration = {
|
||||
async up(db: Kysely<unknown>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('visit_reports')
|
||||
.addColumn('key', 'text', (column) => column.primaryKey())
|
||||
.addColumn('data', 'text', (column) => column.notNull())
|
||||
.execute();
|
||||
},
|
||||
async down(db: Kysely<unknown>): Promise<void> {
|
||||
await db.schema.dropTable('visit_reports').execute();
|
||||
}
|
||||
};
|
||||
@@ -3,11 +3,13 @@ import { addVisitClassification } from './0002_visit_classification.js';
|
||||
import { visitRetention } from './0003_visit_retention.js';
|
||||
import { removeVisitCollection } from './0004_remove_visit_collection.js';
|
||||
import { addVisitOutcome } from './0005_visit_outcome.js';
|
||||
import { visitReports } from './0006_visit_reports.js';
|
||||
|
||||
export const migrations = {
|
||||
'0001_visits': initial,
|
||||
'0002_visit_classification': addVisitClassification,
|
||||
'0003_visit_retention': visitRetention,
|
||||
'0004_remove_visit_collection': removeVisitCollection,
|
||||
'0005_visit_outcome': addVisitOutcome
|
||||
'0005_visit_outcome': addVisitOutcome,
|
||||
'0006_visit_reports': visitReports
|
||||
};
|
||||
@@ -54,7 +54,8 @@ test('opens a real WAL database and applies versioned migrations once', async ()
|
||||
'0002_visit_classification',
|
||||
'0003_visit_retention',
|
||||
'0004_remove_visit_collection',
|
||||
'0005_visit_outcome'
|
||||
'0005_visit_outcome',
|
||||
'0006_visit_reports'
|
||||
]);
|
||||
const tables = await db.introspection.getTables();
|
||||
const names = tables.map((table) => table.name);
|
||||
@@ -130,7 +131,8 @@ test('adds request-time labels without changing existing visit data', async () =
|
||||
'0002_visit_classification',
|
||||
'0003_visit_retention',
|
||||
'0004_remove_visit_collection',
|
||||
'0005_visit_outcome'
|
||||
'0005_visit_outcome',
|
||||
'0006_visit_reports'
|
||||
]);
|
||||
expect(await db.selectFrom('visits').selectAll().executeTakeFirstOrThrow()).toEqual({
|
||||
...visit,
|
||||
@@ -168,7 +170,8 @@ test('a failed migration rolls back schema and bookkeeping so startup can retry'
|
||||
'0002_visit_classification',
|
||||
'0003_visit_retention',
|
||||
'0004_remove_visit_collection',
|
||||
'0005_visit_outcome'
|
||||
'0005_visit_outcome',
|
||||
'0006_visit_reports'
|
||||
]);
|
||||
expect(await runMigrations(db)).toEqual([]);
|
||||
} finally {
|
||||
|
||||
@@ -15,8 +15,14 @@ export interface VisitTable {
|
||||
os: string;
|
||||
}
|
||||
|
||||
export interface VisitReportTable {
|
||||
key: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface Database {
|
||||
visits: VisitTable;
|
||||
visit_reports: VisitReportTable;
|
||||
}
|
||||
|
||||
export type ClassifiedVisit = Omit<Insertable<VisitTable>, 'id'>;
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
export { aggregateVisitReports } from './queries.js';
|
||||
export { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js';
|
||||
export { visitReportKey, zeroVisitCounts } from './types.js';
|
||||
export type {
|
||||
AggregatedVisitReports,
|
||||
DailyVisitCounts,
|
||||
FamilyVisitCounts,
|
||||
PreparedVisitReport,
|
||||
ReportInventory,
|
||||
ScopedFamilyCounts,
|
||||
SerializedVisitReport,
|
||||
StoredVisitReports,
|
||||
TimeBucket,
|
||||
VisitCounts,
|
||||
VisitReportPeriod
|
||||
} from './types.js';
|
||||
@@ -0,0 +1,151 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'vitest';
|
||||
import { openDb, runMigrations, type ClassifiedVisit } from '../index.js';
|
||||
import { aggregateVisitReports } from './queries.js';
|
||||
import { visitReportKey, type ReportInventory } from './types.js';
|
||||
|
||||
let db: Awaited<ReturnType<typeof openDb>>;
|
||||
|
||||
const domain = 'go.example.test';
|
||||
const otherDomain = 'other.example.test';
|
||||
const since = Date.UTC(2026, 0, 31, 23, 59, 59, 999);
|
||||
const nextDay = Date.UTC(2026, 1, 1);
|
||||
const asOf = Date.UTC(2026, 1, 2);
|
||||
const visit: ClassifiedVisit = {
|
||||
domain,
|
||||
path: '/docs',
|
||||
outcome: 'valid',
|
||||
visited_at: since,
|
||||
ip: '127.0.0.1',
|
||||
user_agent: null,
|
||||
accept_language: null,
|
||||
browser: 'Firefox',
|
||||
os: 'Linux'
|
||||
};
|
||||
const inventory: ReportInventory = [
|
||||
{ domain, paths: ['/docs', '/new', '/docs'] },
|
||||
{ domain: otherDomain, paths: ['/docs'] },
|
||||
{ domain: 'empty.example.test', paths: [] }
|
||||
];
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await openDb(':memory:');
|
||||
await runMigrations(db);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await db.destroy();
|
||||
});
|
||||
|
||||
async function seedVisits(): Promise<void> {
|
||||
await db
|
||||
.insertInto('visits')
|
||||
.values([
|
||||
visit,
|
||||
{ ...visit, path: '/', visited_at: nextDay, browser: 'Safari', os: 'macOS' },
|
||||
{ ...visit, path: '/', outcome: 'invalid', visited_at: nextDay + 1 },
|
||||
{
|
||||
...visit,
|
||||
path: '/unconfigured',
|
||||
outcome: 'invalid',
|
||||
visited_at: nextDay + 2,
|
||||
browser: 'Other',
|
||||
os: 'Other'
|
||||
},
|
||||
{
|
||||
...visit,
|
||||
domain: 'removed.example.test',
|
||||
path: '/',
|
||||
outcome: 'invalid',
|
||||
visited_at: nextDay + 3
|
||||
},
|
||||
{
|
||||
...visit,
|
||||
domain: otherDomain,
|
||||
outcome: 'invalid',
|
||||
visited_at: nextDay + 4,
|
||||
os: 'Windows'
|
||||
},
|
||||
{ ...visit, visited_at: since - 1 },
|
||||
{ ...visit, visited_at: asOf }
|
||||
])
|
||||
.execute();
|
||||
}
|
||||
|
||||
test('aggregates UTC days and stored families for overview, current domains and known paths', async () => {
|
||||
await seedVisits();
|
||||
const result = await aggregateVisitReports(db, since, asOf, inventory);
|
||||
const overview = visitReportKey();
|
||||
const domainScope = visitReportKey(domain);
|
||||
const docs = visitReportKey(domain, '/docs');
|
||||
const root = visitReportKey(domain, '/');
|
||||
const otherDomainScope = visitReportKey(otherDomain);
|
||||
const otherDocs = visitReportKey(otherDomain, '/docs');
|
||||
|
||||
expect(result.daily).toHaveLength(8);
|
||||
expect(result.daily).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ scope: overview, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
|
||||
{ scope: overview, day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 },
|
||||
{ scope: domainScope, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
|
||||
{ scope: domainScope, day: '2026-02-01', total: 3, valid: 1, invalid: 2, base: 2 },
|
||||
{ scope: docs, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
|
||||
{ scope: root, day: '2026-02-01', total: 2, valid: 1, invalid: 1, base: 2 },
|
||||
{ scope: otherDomainScope, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 },
|
||||
{ scope: otherDocs, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 }
|
||||
])
|
||||
);
|
||||
expect(result.browsers.filter(({ scope }) => scope === overview)).toEqual([
|
||||
{ scope: overview, name: 'Firefox', total: 4, valid: 1, invalid: 3, base: 2 },
|
||||
{ scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
|
||||
{ scope: overview, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
|
||||
]);
|
||||
expect(result.operatingSystems.filter(({ scope }) => scope === overview)).toEqual([
|
||||
{ scope: overview, name: 'Linux', total: 3, valid: 1, invalid: 2, base: 2 },
|
||||
{ scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
|
||||
{ scope: overview, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 },
|
||||
{ scope: overview, name: 'macOS', total: 1, valid: 1, invalid: 0, base: 1 }
|
||||
]);
|
||||
expect(result.browsers.filter(({ scope }) => scope === domainScope)).toEqual([
|
||||
{ scope: domainScope, name: 'Firefox', total: 2, valid: 1, invalid: 1, base: 1 },
|
||||
{ scope: domainScope, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 },
|
||||
{ scope: domainScope, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
|
||||
]);
|
||||
expect(result.browsers.filter(({ scope }) => scope === root)).toEqual([
|
||||
{ scope: root, name: 'Firefox', total: 1, valid: 0, invalid: 1, base: 1 },
|
||||
{ scope: root, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 }
|
||||
]);
|
||||
expect(result.operatingSystems.filter(({ scope }) => scope === otherDocs)).toEqual([
|
||||
{ scope: otherDocs, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 }
|
||||
]);
|
||||
const familyScopes = new Set(
|
||||
[...result.browsers, ...result.operatingSystems].map(({ scope }) => scope)
|
||||
);
|
||||
expect(familyScopes).toEqual(
|
||||
new Set([overview, domainScope, docs, root, otherDomainScope, otherDocs])
|
||||
);
|
||||
});
|
||||
|
||||
test('empty inventory still aggregates historical visits only under overview', async () => {
|
||||
await seedVisits();
|
||||
const result = await aggregateVisitReports(db, since, asOf, []);
|
||||
|
||||
expect(result.daily).toEqual([
|
||||
{ scope: 'overview', day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 },
|
||||
{ scope: 'overview', day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 }
|
||||
]);
|
||||
expect(result.browsers.map(({ scope }) => scope)).toEqual(['overview', 'overview', 'overview']);
|
||||
expect(result.operatingSystems.map(({ scope }) => scope)).toEqual([
|
||||
'overview',
|
||||
'overview',
|
||||
'overview',
|
||||
'overview'
|
||||
]);
|
||||
});
|
||||
|
||||
test('no visits produces no aggregate rows for configured scopes', async () => {
|
||||
expect(await aggregateVisitReports(db, since, asOf, inventory)).toEqual({
|
||||
daily: [],
|
||||
browsers: [],
|
||||
operatingSystems: []
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import { sql, type Kysely } from 'kysely';
|
||||
import type { Database } from '../types.js';
|
||||
import {
|
||||
visitReportKey,
|
||||
type AggregatedVisitReports,
|
||||
type DailyVisitCounts,
|
||||
type ReportInventory,
|
||||
type ScopedFamilyCounts
|
||||
} from './types.js';
|
||||
|
||||
export async function aggregateVisitReports(
|
||||
db: Kysely<Database>,
|
||||
since: number,
|
||||
asOf: number,
|
||||
inventory: ReportInventory
|
||||
): Promise<AggregatedVisitReports> {
|
||||
const scopes = scopedVisits(since, asOf, inventory);
|
||||
const counts = visitCountColumns();
|
||||
|
||||
return db.transaction().execute(async (transaction) => {
|
||||
const daily = await sql<DailyVisitCounts>`
|
||||
${scopes}
|
||||
SELECT scope, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts}
|
||||
FROM scoped_visits
|
||||
GROUP BY scope, day
|
||||
ORDER BY scope, day
|
||||
`.execute(transaction);
|
||||
const browsers = await sql<ScopedFamilyCounts>`
|
||||
${scopes}
|
||||
SELECT scope, browser AS name, ${counts}
|
||||
FROM scoped_visits
|
||||
GROUP BY scope, browser
|
||||
ORDER BY scope, name
|
||||
`.execute(transaction);
|
||||
const operatingSystems = await sql<ScopedFamilyCounts>`
|
||||
${scopes}
|
||||
SELECT scope, os AS name, ${counts}
|
||||
FROM scoped_visits
|
||||
GROUP BY scope, os
|
||||
ORDER BY scope, name
|
||||
`.execute(transaction);
|
||||
|
||||
return {
|
||||
daily: daily.rows,
|
||||
browsers: browsers.rows,
|
||||
operatingSystems: operatingSystems.rows
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function scopedVisits(since: number, asOf: number, inventory: ReportInventory) {
|
||||
const domains = [...new Set(inventory.map(({ domain }) => domain))];
|
||||
const currentDomain = domains.length === 0 ? sql`0` : sql`domain IN (${sql.join(domains)})`;
|
||||
const linkRules = inventory.map(({ domain, paths }) => {
|
||||
const currentPaths = [...new Set([...paths, '/'])];
|
||||
return sql`(domain = ${domain} AND path IN (${sql.join(currentPaths)}))`;
|
||||
});
|
||||
const currentLink = linkRules.length === 0 ? sql`0` : sql.join(linkRules, sql` OR `);
|
||||
|
||||
return sql`
|
||||
WITH retained_visits AS (
|
||||
SELECT domain, path, outcome, visited_at, browser, os
|
||||
FROM visits
|
||||
WHERE visited_at >= ${since} AND visited_at < ${asOf}
|
||||
), scoped_visits AS (
|
||||
SELECT ${visitReportKey()} AS scope, path, outcome, visited_at, browser, os
|
||||
FROM retained_visits
|
||||
UNION ALL
|
||||
SELECT json_array(domain) AS scope, path, outcome, visited_at, browser, os
|
||||
FROM retained_visits
|
||||
WHERE ${currentDomain}
|
||||
UNION ALL
|
||||
SELECT json_array(domain, path) AS scope, path, outcome, visited_at, browser, os
|
||||
FROM retained_visits
|
||||
WHERE ${currentLink}
|
||||
)
|
||||
`;
|
||||
}
|
||||
|
||||
function visitCountColumns() {
|
||||
return sql`
|
||||
COUNT(*) AS total,
|
||||
SUM(CASE WHEN outcome = ${'valid'} THEN 1 ELSE 0 END) AS valid,
|
||||
SUM(CASE WHEN outcome = ${'invalid'} THEN 1 ELSE 0 END) AS invalid,
|
||||
SUM(CASE WHEN path = ${'/'} THEN 1 ELSE 0 END) AS base
|
||||
`;
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { afterEach, beforeEach, expect, test } from 'vitest';
|
||||
import { openDb, runMigrations } from '../index.js';
|
||||
import { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js';
|
||||
import {
|
||||
visitReportKey,
|
||||
zeroVisitCounts,
|
||||
type PreparedVisitReport,
|
||||
type SerializedVisitReport,
|
||||
type VisitCounts,
|
||||
type VisitReportPeriod
|
||||
} from './types.js';
|
||||
|
||||
let db: Awaited<ReturnType<typeof openDb>>;
|
||||
|
||||
const domain = 'go.example.test';
|
||||
const period: VisitReportPeriod = {
|
||||
timezone: 'UTC',
|
||||
retentionMonths: 3,
|
||||
since: Date.UTC(2026, 0, 1),
|
||||
asOf: Date.UTC(2026, 0, 2),
|
||||
days: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 0, 2), partial: false }],
|
||||
weeks: [{ start: Date.UTC(2025, 11, 29), end: Date.UTC(2026, 0, 5), partial: true }],
|
||||
months: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 1, 1), partial: true }]
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
db = await openDb(':memory:');
|
||||
await runMigrations(db);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await db.destroy();
|
||||
});
|
||||
|
||||
function preparedReport(counts: VisitCounts = zeroVisitCounts()): PreparedVisitReport {
|
||||
return {
|
||||
totals: counts,
|
||||
daily: [counts],
|
||||
weekly: [counts],
|
||||
monthly: [counts],
|
||||
browsers: [],
|
||||
operatingSystems: [],
|
||||
domainCounts: [],
|
||||
linkCounts: []
|
||||
};
|
||||
}
|
||||
|
||||
function serializedReport(key: string, report: PreparedVisitReport): SerializedVisitReport {
|
||||
return { key, data: JSON.stringify(report) };
|
||||
}
|
||||
|
||||
test('reads the shared period, scope report and selected path from prepared rows', async () => {
|
||||
const selection = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
|
||||
selection.browsers = [{ name: 'Firefox', counts: selection.totals }];
|
||||
selection.operatingSystems = [{ name: 'Linux', counts: selection.totals }];
|
||||
const domainReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 1 });
|
||||
domainReport.linkCounts = [{ path: '/docs', counts: selection.totals }];
|
||||
const overview = preparedReport({ total: 3, valid: 2, invalid: 1, base: 1 });
|
||||
overview.domainCounts = [{ domain, counts: domainReport.totals }];
|
||||
await publishVisitReports(db, period, [
|
||||
serializedReport('empty', preparedReport()),
|
||||
serializedReport(visitReportKey(), overview),
|
||||
serializedReport(visitReportKey(domain), domainReport),
|
||||
serializedReport(visitReportKey(domain, '/docs'), selection)
|
||||
]);
|
||||
|
||||
expect(await readPreparedVisitReports(db)).toEqual({
|
||||
period,
|
||||
report: overview,
|
||||
selection: overview
|
||||
});
|
||||
expect(await readPreparedVisitReports(db, domain)).toEqual({
|
||||
period,
|
||||
report: domainReport,
|
||||
selection: domainReport
|
||||
});
|
||||
expect(await readPreparedVisitReports(db, domain, '/docs')).toEqual({
|
||||
period,
|
||||
report: domainReport,
|
||||
selection
|
||||
});
|
||||
});
|
||||
|
||||
test('publication replaces all output rows and a failed replacement preserves the previous set', async () => {
|
||||
const oldReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
|
||||
await publishVisitReports(db, period, [
|
||||
serializedReport(visitReportKey(), oldReport),
|
||||
serializedReport(visitReportKey(domain), oldReport)
|
||||
]);
|
||||
const replacementPeriod = { ...period, retentionMonths: 9 as const };
|
||||
const replacementReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 0 });
|
||||
const replacementRows = [serializedReport(visitReportKey(), replacementReport)];
|
||||
await publishVisitReports(db, replacementPeriod, replacementRows);
|
||||
|
||||
expect(await db.selectFrom('visit_reports').select('key').orderBy('key').execute()).toEqual([
|
||||
{ key: 'overview' },
|
||||
{ key: 'period' }
|
||||
]);
|
||||
expect(await readPreparedVisitReports(db, domain)).toBeNull();
|
||||
await expect(
|
||||
publishVisitReports(db, period, [
|
||||
serializedReport(visitReportKey(), oldReport),
|
||||
serializedReport(visitReportKey(), oldReport)
|
||||
])
|
||||
).rejects.toThrow();
|
||||
expect(await readPreparedVisitReports(db)).toEqual({
|
||||
period: replacementPeriod,
|
||||
report: replacementReport,
|
||||
selection: replacementReport
|
||||
});
|
||||
});
|
||||
|
||||
test('a published zero-count path uses the empty template but a missing domain stays unavailable', async () => {
|
||||
const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 1 });
|
||||
domainReport.linkCounts = [{ path: '/new', counts: zeroVisitCounts() }];
|
||||
const empty = preparedReport();
|
||||
await publishVisitReports(db, period, [
|
||||
serializedReport(visitReportKey(domain), domainReport),
|
||||
serializedReport('empty', empty)
|
||||
]);
|
||||
|
||||
expect(await readPreparedVisitReports(db, domain, '/new')).toEqual({
|
||||
period,
|
||||
report: domainReport,
|
||||
selection: empty
|
||||
});
|
||||
expect(await readPreparedVisitReports(db, 'new.example.test', '/new')).toBeNull();
|
||||
});
|
||||
|
||||
test('missing link bodies stay unavailable for unpublished paths and populated links', async () => {
|
||||
const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 });
|
||||
domainReport.linkCounts = [{ path: '/docs', counts: domainReport.totals }];
|
||||
await publishVisitReports(db, period, [
|
||||
serializedReport(visitReportKey(domain), domainReport),
|
||||
serializedReport('empty', preparedReport())
|
||||
]);
|
||||
|
||||
expect(await readPreparedVisitReports(db, domain, '/new')).toBeNull();
|
||||
expect(await readPreparedVisitReports(db, domain, '/docs')).toBeNull();
|
||||
});
|
||||
|
||||
test('missing metadata is unavailable and clearing deletes every output row', async () => {
|
||||
expect(await readPreparedVisitReports(db)).toBeNull();
|
||||
await db
|
||||
.insertInto('visit_reports')
|
||||
.values(serializedReport(visitReportKey(), preparedReport()))
|
||||
.execute();
|
||||
expect(await readPreparedVisitReports(db)).toBeNull();
|
||||
|
||||
await publishVisitReports(db, period, [
|
||||
serializedReport(visitReportKey(), preparedReport()),
|
||||
serializedReport('empty', preparedReport())
|
||||
]);
|
||||
await clearVisitReports(db);
|
||||
expect(await db.selectFrom('visit_reports').selectAll().execute()).toEqual([]);
|
||||
expect(await readPreparedVisitReports(db)).toBeNull();
|
||||
});
|
||||
|
||||
test('malformed stored JSON is left for the caller to handle', async () => {
|
||||
await publishVisitReports(db, period, [{ key: visitReportKey(), data: '{' }]);
|
||||
|
||||
await expect(readPreparedVisitReports(db)).rejects.toThrow(SyntaxError);
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { Kysely } from 'kysely';
|
||||
import type { Database } from '../types.js';
|
||||
import {
|
||||
visitReportKey,
|
||||
type PreparedVisitReport,
|
||||
type SerializedVisitReport,
|
||||
type StoredVisitReports,
|
||||
type VisitReportPeriod
|
||||
} from './types.js';
|
||||
|
||||
export async function clearVisitReports(db: Kysely<Database>): Promise<void> {
|
||||
await db.deleteFrom('visit_reports').execute();
|
||||
}
|
||||
|
||||
export async function publishVisitReports(
|
||||
db: Kysely<Database>,
|
||||
period: VisitReportPeriod,
|
||||
reports: SerializedVisitReport[]
|
||||
): Promise<void> {
|
||||
const rows = [{ key: 'period', data: JSON.stringify(period) }, ...reports];
|
||||
|
||||
await db.transaction().execute(async (transaction) => {
|
||||
await transaction.deleteFrom('visit_reports').execute();
|
||||
await transaction.insertInto('visit_reports').values(rows).execute();
|
||||
});
|
||||
}
|
||||
|
||||
export async function readPreparedVisitReports(
|
||||
db: Kysely<Database>,
|
||||
domain?: string,
|
||||
path?: string
|
||||
): Promise<StoredVisitReports | null> {
|
||||
const reportKey = visitReportKey(domain);
|
||||
const selectionKey = visitReportKey(domain, path);
|
||||
const keys = [...new Set(['period', reportKey, selectionKey, 'empty'])];
|
||||
const rows = await db.selectFrom('visit_reports').selectAll().where('key', 'in', keys).execute();
|
||||
const dataByKey = new Map(rows.map(({ key, data }) => [key, data]));
|
||||
const periodData = dataByKey.get('period');
|
||||
const reportData = dataByKey.get(reportKey);
|
||||
if (periodData === undefined || reportData === undefined) return null;
|
||||
|
||||
const report = JSON.parse(reportData) as PreparedVisitReport;
|
||||
let selection = report;
|
||||
if (selectionKey !== reportKey) {
|
||||
let selectionData = dataByKey.get(selectionKey);
|
||||
if (selectionData === undefined) {
|
||||
const linkCounts = report.linkCounts.find((link) => link.path === path)?.counts;
|
||||
if (linkCounts?.total !== 0) return null;
|
||||
selectionData = dataByKey.get('empty');
|
||||
}
|
||||
if (selectionData === undefined) return null;
|
||||
selection = JSON.parse(selectionData) as PreparedVisitReport;
|
||||
}
|
||||
return {
|
||||
period: JSON.parse(periodData) as VisitReportPeriod,
|
||||
report,
|
||||
selection
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { expect, test } from 'vitest';
|
||||
import { visitReportKey, zeroVisitCounts } from './types.js';
|
||||
|
||||
test('zero counts are fresh objects', () => {
|
||||
const first = zeroVisitCounts();
|
||||
const second = zeroVisitCounts();
|
||||
first.total = 1;
|
||||
|
||||
expect(second).toEqual({ total: 0, valid: 0, invalid: 0, base: 0 });
|
||||
expect(first).not.toBe(second);
|
||||
});
|
||||
|
||||
test('scope keys use overview or JSON-encoded domains and exact paths', () => {
|
||||
expect(visitReportKey()).toBe('overview');
|
||||
expect(visitReportKey('go.example.test')).toBe('["go.example.test"]');
|
||||
expect(visitReportKey('go.example.test', '/')).toBe('["go.example.test","/"]');
|
||||
expect(visitReportKey('go.example.test', '/café,"docs')).toBe(
|
||||
JSON.stringify(['go.example.test', '/café,"docs'])
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
export interface VisitCounts {
|
||||
total: number;
|
||||
valid: number;
|
||||
invalid: number;
|
||||
base: number;
|
||||
}
|
||||
|
||||
export function zeroVisitCounts(): VisitCounts {
|
||||
return { total: 0, valid: 0, invalid: 0, base: 0 };
|
||||
}
|
||||
|
||||
export interface TimeBucket {
|
||||
start: number;
|
||||
end: number;
|
||||
partial: boolean;
|
||||
}
|
||||
|
||||
export interface VisitReportPeriod {
|
||||
timezone: 'UTC';
|
||||
retentionMonths: 3 | 9 | 12;
|
||||
since: number;
|
||||
asOf: number;
|
||||
days: TimeBucket[];
|
||||
weeks: TimeBucket[];
|
||||
months: TimeBucket[];
|
||||
}
|
||||
|
||||
export interface FamilyVisitCounts {
|
||||
name: string;
|
||||
counts: VisitCounts;
|
||||
}
|
||||
|
||||
export interface PreparedVisitReport {
|
||||
totals: VisitCounts;
|
||||
daily: VisitCounts[];
|
||||
weekly: VisitCounts[];
|
||||
monthly: VisitCounts[];
|
||||
browsers: FamilyVisitCounts[];
|
||||
operatingSystems: FamilyVisitCounts[];
|
||||
domainCounts: Array<{ domain: string; counts: VisitCounts }>;
|
||||
linkCounts: Array<{ path: string; counts: VisitCounts }>;
|
||||
}
|
||||
|
||||
export type ReportInventory = Array<{ domain: string; paths: string[] }>;
|
||||
|
||||
export function visitReportKey(domain?: string, path?: string): string {
|
||||
if (domain === undefined) return 'overview';
|
||||
return JSON.stringify(path === undefined ? [domain] : [domain, path]);
|
||||
}
|
||||
|
||||
export interface SerializedVisitReport {
|
||||
key: string;
|
||||
data: string;
|
||||
}
|
||||
|
||||
export interface StoredVisitReports {
|
||||
period: VisitReportPeriod;
|
||||
report: PreparedVisitReport;
|
||||
selection: PreparedVisitReport;
|
||||
}
|
||||
|
||||
export interface DailyVisitCounts extends VisitCounts {
|
||||
scope: string;
|
||||
day: string;
|
||||
}
|
||||
|
||||
export interface ScopedFamilyCounts extends VisitCounts {
|
||||
scope: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface AggregatedVisitReports {
|
||||
daily: DailyVisitCounts[];
|
||||
browsers: ScopedFamilyCounts[];
|
||||
operatingSystems: ScopedFamilyCounts[];
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Synthetic profiles use the existing approximate browser/OS family names.
|
||||
const profiles = [
|
||||
{
|
||||
weight: 36,
|
||||
browser: 'Chrome',
|
||||
os: 'Windows',
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/134.0 Safari/537.36'
|
||||
},
|
||||
{
|
||||
weight: 20,
|
||||
browser: 'Chrome',
|
||||
os: 'Android',
|
||||
userAgent: 'Mozilla/5.0 (Linux; Android 15) Chrome/134.0 Mobile Safari/537.36'
|
||||
},
|
||||
{
|
||||
weight: 14,
|
||||
browser: 'Safari',
|
||||
os: 'macOS',
|
||||
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) Version/17.0 Safari/605.1.15'
|
||||
},
|
||||
{
|
||||
weight: 10,
|
||||
browser: 'Safari',
|
||||
os: 'iOS',
|
||||
userAgent:
|
||||
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) Version/17.0 Mobile Safari/604.1'
|
||||
},
|
||||
{
|
||||
weight: 10,
|
||||
browser: 'Firefox',
|
||||
os: 'Linux',
|
||||
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) Firefox/136.0'
|
||||
},
|
||||
{
|
||||
weight: 5,
|
||||
browser: 'Edge',
|
||||
os: 'Windows',
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/134.0 Safari/537.36 Edg/134.0'
|
||||
},
|
||||
{
|
||||
weight: 2,
|
||||
browser: 'Opera',
|
||||
os: 'Windows',
|
||||
userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/134.0 Safari/537.36 OPR/118.0'
|
||||
},
|
||||
{
|
||||
weight: 2,
|
||||
browser: 'Chrome',
|
||||
os: 'ChromeOS',
|
||||
userAgent: 'Mozilla/5.0 (X11; CrOS x86_64 16000.0.0) Chrome/134.0 Safari/537.36'
|
||||
},
|
||||
{ weight: 1, browser: 'Other', os: 'Other', userAgent: 'SyntheticClient/1.0' }
|
||||
];
|
||||
|
||||
const languages = [
|
||||
'en-US,en;q=0.9',
|
||||
'en-GB,en;q=0.9',
|
||||
'de-DE,de;q=0.9',
|
||||
'fr-FR,fr;q=0.9',
|
||||
'ja-JP,ja;q=0.9'
|
||||
];
|
||||
|
||||
export function seedWindowStart(months, asOf) {
|
||||
// Retention is UTC calendar months, including month-end clamping.
|
||||
const date = new Date(asOf);
|
||||
const day = date.getUTCDate();
|
||||
date.setUTCDate(1);
|
||||
date.setUTCMonth(date.getUTCMonth() - months);
|
||||
const lastDay = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 0)).getUTCDate();
|
||||
date.setUTCDate(Math.min(day, lastDay));
|
||||
return date.getTime();
|
||||
}
|
||||
|
||||
export function* generateVisits({ domains, visitsPerDomain, since, asOf, seed }) {
|
||||
const random = seededRandom(seed);
|
||||
for (const [domain, prepared] of Object.entries(domains)) {
|
||||
const paths = requestPaths(prepared.links);
|
||||
for (let index = 0; index < visitsPerDomain; index++) {
|
||||
const visitedAt = visitTime(random, since, asOf);
|
||||
const path = weightedChoice(random, paths).path;
|
||||
const profile = weightedChoice(random, profiles);
|
||||
const languageIndex = Math.floor(random() * languages.length);
|
||||
yield {
|
||||
domain,
|
||||
path,
|
||||
outcome: visitOutcome(prepared.links[path], visitedAt),
|
||||
visited_at: visitedAt,
|
||||
ip: '192.0.2.1',
|
||||
user_agent: `${profile.userAgent} GotoSeed/${seed}`,
|
||||
accept_language: languages[languageIndex],
|
||||
browser: profile.browser,
|
||||
os: profile.os
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function requestPaths(links) {
|
||||
let missingPath = '/__seed-missing__';
|
||||
while (Object.hasOwn(links, missingPath)) missingPath += '-';
|
||||
const paths = [
|
||||
{ path: '/', weight: 10 },
|
||||
{ path: missingPath, weight: 10 }
|
||||
];
|
||||
// Prefer configured links, with a separate share for base and unmatched requests.
|
||||
const configured = Object.keys(links);
|
||||
const popularity = configured.reduce((total, _path, index) => total + 1 / (index + 1), 0);
|
||||
for (const [index, path] of configured.entries()) {
|
||||
const weight = 80 / ((index + 1) * popularity);
|
||||
paths.push({ path, weight });
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
function visitOutcome(link, visitedAt) {
|
||||
if (!link || !link.enabled) return 'invalid';
|
||||
if (link.valid_from !== undefined && visitedAt < Date.parse(link.valid_from)) return 'invalid';
|
||||
if (link.valid_until !== undefined && visitedAt >= Date.parse(link.valid_until)) return 'invalid';
|
||||
return 'valid';
|
||||
}
|
||||
|
||||
function visitTime(random, since, asOf) {
|
||||
while (true) {
|
||||
const timestamp = since + Math.floor(random() * (asOf - since));
|
||||
const date = new Date(timestamp);
|
||||
const weekday = date.getUTCDay();
|
||||
const hour = date.getUTCHours();
|
||||
const weekdayWeight = weekday === 0 || weekday === 6 ? 0.35 : 1;
|
||||
const hourWeight = hour >= 8 && hour < 19 ? 1 : 0.3;
|
||||
const recencyWeight = 0.4 + 0.6 * ((timestamp - since) / (asOf - since));
|
||||
const weight = weekdayWeight * hourWeight * recencyWeight;
|
||||
if (random() < weight) return timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
function weightedChoice(random, entries) {
|
||||
const total = entries.reduce((sum, entry) => sum + entry.weight, 0);
|
||||
let remaining = random() * total;
|
||||
for (const entry of entries) {
|
||||
remaining -= entry.weight;
|
||||
if (remaining < 0) return entry;
|
||||
}
|
||||
return entries.at(-1);
|
||||
}
|
||||
|
||||
function seededRandom(seed) {
|
||||
// Park–Miller: repeatable fixture sampling, not cryptographic randomness.
|
||||
let state = seed;
|
||||
return function random() {
|
||||
state = (state * 48271) % 2147483647;
|
||||
return (state - 1) / 2147483646;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { stat } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import { setImmediate } from 'node:timers/promises';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { loadConfigFile } from '../packages/config/src/index.ts';
|
||||
import { openDb, insertVisit } from '../packages/db/src/index.ts';
|
||||
import { generateVisits, seedWindowStart } from './seed-visits-data.js';
|
||||
|
||||
// Keep write transactions short while the development services share SQLite.
|
||||
const batchSize = 250;
|
||||
const help = `Development-only synthetic visit seeder (append-only).
|
||||
|
||||
Usage:
|
||||
pnpm run seed:visits --config <path> [options]
|
||||
|
||||
Options:
|
||||
--config <path> YAML configuration; required
|
||||
--visits-per-domain <count> Visits for each configured domain (default: 10000)
|
||||
--seed <integer> Repeatable sampling seed, 1–2147483646 (default: 42)
|
||||
--apply Append visits; otherwise preview without opening SQLite
|
||||
--help Show this help
|
||||
|
||||
Uses the configured visits database and retention window. The database must
|
||||
already exist with the current visits schema; this tool never runs migrations.
|
||||
Rerunning with --apply appends another dataset. Back up your development data
|
||||
before applying. Synthetic rows use IP 192.0.2.1 and a GotoSeed/<seed> UA marker.
|
||||
`;
|
||||
|
||||
async function main() {
|
||||
const options = readOptions();
|
||||
if (options.help) {
|
||||
console.log(help);
|
||||
return;
|
||||
}
|
||||
const prepared = await loadConfigFile(options.configPath);
|
||||
const plan = seedPlan(prepared, options);
|
||||
printPlan(plan, options.apply);
|
||||
if (!options.apply) return;
|
||||
await appendVisits(plan);
|
||||
console.log(
|
||||
'Prepared reports update at the next worker refresh; this tool does not trigger one.'
|
||||
);
|
||||
}
|
||||
|
||||
function readOptions() {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
config: { type: 'string' },
|
||||
'visits-per-domain': { type: 'string', default: '10000' },
|
||||
seed: { type: 'string', default: '42' },
|
||||
apply: { type: 'boolean', default: false },
|
||||
help: { type: 'boolean', default: false }
|
||||
}
|
||||
});
|
||||
if (values.help) return { help: true };
|
||||
if (!values.config?.trim()) throw new Error('--config is required; use --help for usage');
|
||||
const visitsPerDomain = positiveInteger(values['visits-per-domain'], '--visits-per-domain');
|
||||
const seed = positiveInteger(values.seed, '--seed');
|
||||
if (seed >= 2147483647) throw new Error('--seed must be between 1 and 2147483646');
|
||||
return { configPath: resolve(values.config), visitsPerDomain, seed, apply: values.apply };
|
||||
}
|
||||
|
||||
function positiveInteger(value, name) {
|
||||
const number = Number(value);
|
||||
if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(number)) {
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
return number;
|
||||
}
|
||||
|
||||
function seedPlan(prepared, options) {
|
||||
const { configuration, domains } = prepared;
|
||||
const months = configuration.analytics.retention_months;
|
||||
if (months === 0) throw new Error('Analytics collection is disabled; no visits will be seeded');
|
||||
if (Object.keys(domains).length === 0) throw new Error('No configured domains to seed');
|
||||
const total = Object.keys(domains).length * options.visitsPerDomain;
|
||||
if (!Number.isSafeInteger(total))
|
||||
throw new Error('Total visit count exceeds the safe integer range');
|
||||
const databasePath = configuration.database.path;
|
||||
if (databasePath === configuration.database.worker_path) {
|
||||
throw new Error('Visits and worker database paths must differ');
|
||||
}
|
||||
const asOf = Date.now();
|
||||
const since = seedWindowStart(months, asOf);
|
||||
return {
|
||||
databasePath,
|
||||
domains,
|
||||
since,
|
||||
asOf,
|
||||
visitsPerDomain: options.visitsPerDomain,
|
||||
seed: options.seed,
|
||||
total
|
||||
};
|
||||
}
|
||||
|
||||
function printPlan(plan, apply) {
|
||||
console.log(apply ? 'Append synthetic visits' : 'Dry run — no database opened or changed');
|
||||
console.log(`Database: ${plan.databasePath}`);
|
||||
console.log(
|
||||
`UTC window: ${new Date(plan.since).toISOString()} to ${new Date(plan.asOf).toISOString()} (exclusive)`
|
||||
);
|
||||
console.log(`Sampling seed: ${plan.seed} (timestamps are relative to this run)`);
|
||||
for (const domain of Object.keys(plan.domains))
|
||||
console.log(` ${domain}: ${plan.visitsPerDomain} visits`);
|
||||
console.log(`Total: ${plan.total} visits. Reruns append; existing data is not replaced.`);
|
||||
if (!apply)
|
||||
console.log('Review the target and back up your development data, then repeat with --apply.');
|
||||
}
|
||||
|
||||
async function appendVisits(plan) {
|
||||
const file = await stat(plan.databasePath).catch(() => {
|
||||
throw new Error(
|
||||
'Cannot access the existing visits database; start the application normally before seeding'
|
||||
);
|
||||
});
|
||||
if (!file.isFile()) throw new Error('Configured visits database is not a regular file');
|
||||
const db = await openDb(plan.databasePath);
|
||||
let inserted = 0;
|
||||
try {
|
||||
// Validate insert columns before committing any rows, without changing schema.
|
||||
const columns = [
|
||||
'domain',
|
||||
'path',
|
||||
'outcome',
|
||||
'visited_at',
|
||||
'ip',
|
||||
'user_agent',
|
||||
'accept_language',
|
||||
'browser',
|
||||
'os'
|
||||
];
|
||||
await db.selectFrom('visits').select(columns).limit(0).execute();
|
||||
let batch = [];
|
||||
for (const visit of generateVisits(plan)) {
|
||||
batch.push(visit);
|
||||
if (batch.length === batchSize) {
|
||||
await appendBatch(db, batch);
|
||||
inserted += batch.length;
|
||||
batch = [];
|
||||
await setImmediate();
|
||||
}
|
||||
}
|
||||
if (batch.length > 0) {
|
||||
await appendBatch(db, batch);
|
||||
inserted += batch.length;
|
||||
}
|
||||
console.log(`Appended ${inserted} synthetic visits.`);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`Stopped after ${inserted} committed visits; completed batches remain in the database.`
|
||||
);
|
||||
throw error;
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
async function appendBatch(db, visits) {
|
||||
await db.transaction().execute(async (transaction) => {
|
||||
for (const visit of visits) await insertVisit(transaction, visit);
|
||||
});
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
main().catch((error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import { test } from 'node:test';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { validateConfig } from '../packages/config/src/index.ts';
|
||||
import { openDb, insertVisit, runMigrations } from '../packages/db/src/index.ts';
|
||||
import { classifyUserAgent } from '../packages/db/src/classify-user-agent.ts';
|
||||
import { generateVisits, seedWindowStart } from './seed-visits-data.js';
|
||||
|
||||
const script = fileURLToPath(new URL('./seed-visits.js', import.meta.url));
|
||||
const loader = fileURLToPath(new URL('../tools/register.mjs', import.meta.url));
|
||||
const fixture = {
|
||||
site: { url: 'https://admin.example.test' },
|
||||
auth: { credentials: { username: 'test', password: 'seed-test-only-password' } },
|
||||
domains: {
|
||||
'one.example.test': { links: { '/docs': { url: 'https://example.org/docs' } } },
|
||||
'two.example.test': { links: { '/docs': { url: 'https://example.com/docs' } } },
|
||||
'three.example.test': { links: { '/docs': { url: 'https://example.net/docs' } } }
|
||||
}
|
||||
};
|
||||
|
||||
function run(...args) {
|
||||
return spawnSync(process.execPath, ['--import', loader, script, ...args], {
|
||||
encoding: 'utf8',
|
||||
timeout: 15_000
|
||||
});
|
||||
}
|
||||
|
||||
async function deployment(directory) {
|
||||
await mkdir(join(directory, 'config'));
|
||||
const configPath = join(directory, 'config/goto.yaml');
|
||||
await writeFile(configPath, JSON.stringify(fixture));
|
||||
return { configPath, databasePath: join(directory, 'data/goto.sqlite') };
|
||||
}
|
||||
|
||||
test('calendar window clamps month ends and keeps UTC time', () => {
|
||||
const since = seedWindowStart(3, Date.parse('2026-05-31T12:34:56.789Z'));
|
||||
assert.equal(new Date(since).toISOString(), '2026-02-28T12:34:56.789Z');
|
||||
const leap = seedWindowStart(12, Date.parse('2024-02-29T12:00:00Z'));
|
||||
assert.equal(new Date(leap).toISOString(), '2023-02-28T12:00:00.000Z');
|
||||
});
|
||||
|
||||
test('fixed inputs produce repeatable retained traffic with request-time outcomes and consistent profiles', () => {
|
||||
const source = structuredClone(fixture);
|
||||
source.domains['one.example.test'].links = {
|
||||
'/': { url: 'https://example.org/' },
|
||||
'/docs': {
|
||||
url: 'https://example.org/docs',
|
||||
valid_from: '2026-03-31T00:00:00Z',
|
||||
valid_until: '2026-04-30T00:00:00Z'
|
||||
},
|
||||
'/disabled': { url: 'https://example.org/off', enabled: false },
|
||||
'/future': { url: 'https://example.org/future', valid_from: '2027-01-01T00:00:00Z' }
|
||||
};
|
||||
const { domains } = validateConfig(source, { env: {} });
|
||||
const asOf = Date.parse('2026-05-31T12:00:00Z');
|
||||
const since = seedWindowStart(3, asOf);
|
||||
const options = { domains, since, asOf, visitsPerDomain: 300, seed: 42 };
|
||||
const visits = [...generateVisits(options)];
|
||||
assert.deepEqual([...generateVisits(options)], visits);
|
||||
assert.equal(visits.length, 900);
|
||||
assert(visits.some((visit) => visit.path === '/' && visit.outcome === 'valid'));
|
||||
assert(visits.some((visit) => visit.path === '/__seed-missing__' && visit.outcome === 'invalid'));
|
||||
assert(visits.some((visit) => visit.path === '/disabled' && visit.outcome === 'invalid'));
|
||||
const scheduledVisits = visits.filter(
|
||||
(visit) => visit.domain === 'one.example.test' && visit.path === '/docs'
|
||||
);
|
||||
const startsAt = Date.parse(source.domains['one.example.test'].links['/docs'].valid_from);
|
||||
const endsAt = Date.parse(source.domains['one.example.test'].links['/docs'].valid_until);
|
||||
assert(scheduledVisits.some((visit) => visit.visited_at < startsAt));
|
||||
assert(
|
||||
scheduledVisits.some((visit) => visit.visited_at >= startsAt && visit.visited_at < endsAt)
|
||||
);
|
||||
assert(scheduledVisits.some((visit) => visit.visited_at >= endsAt));
|
||||
for (const visit of scheduledVisits) {
|
||||
const active = visit.visited_at >= startsAt && visit.visited_at < endsAt;
|
||||
assert.equal(visit.outcome, active ? 'valid' : 'invalid');
|
||||
}
|
||||
for (const visit of visits) {
|
||||
assert(visit.visited_at >= since && visit.visited_at < asOf);
|
||||
assert.equal(visit.ip, '192.0.2.1');
|
||||
assert(visit.user_agent.endsWith(' GotoSeed/42'));
|
||||
assert.deepEqual(classifyUserAgent(visit.user_agent), { browser: visit.browser, os: visit.os });
|
||||
if (visit.path === '/future') assert.equal(visit.outcome, 'invalid');
|
||||
}
|
||||
});
|
||||
|
||||
test('preview opens no SQLite file and applying refuses to create a missing database', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goto-seed-preview-'));
|
||||
try {
|
||||
const { configPath, databasePath } = await deployment(directory);
|
||||
const preview = run('--config', configPath);
|
||||
assert.equal(preview.status, 0, preview.stderr);
|
||||
assert.match(preview.stdout, /Dry run/);
|
||||
assert.match(preview.stdout, /Total: 30000 visits/);
|
||||
await assert.rejects(access(databasePath), { code: 'ENOENT' });
|
||||
const apply = run('--config', configPath, '--apply');
|
||||
assert.equal(apply.status, 1, apply.stderr);
|
||||
assert.match(apply.stderr, /existing visits database/);
|
||||
await assert.rejects(access(databasePath), { code: 'ENOENT' });
|
||||
await writeFile(configPath, JSON.stringify({ ...fixture, analytics: { retention_months: 0 } }));
|
||||
const disabled = run('--config', configPath, '--apply');
|
||||
assert.equal(disabled.status, 1);
|
||||
assert.match(disabled.stderr, /Analytics collection is disabled/);
|
||||
await assert.rejects(access(databasePath), { code: 'ENOENT' });
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('explicit apply appends all configured domains in bounded batches without replacing existing visits', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goto-seed-apply-'));
|
||||
try {
|
||||
const { configPath, databasePath } = await deployment(directory);
|
||||
const existing = {
|
||||
domain: 'one.example.test',
|
||||
path: '/keep',
|
||||
outcome: 'valid',
|
||||
visited_at: 1,
|
||||
ip: '198.51.100.1',
|
||||
user_agent: 'ExistingClient',
|
||||
accept_language: null,
|
||||
browser: 'Other',
|
||||
os: 'Other'
|
||||
};
|
||||
const writer = await openDb(databasePath);
|
||||
let id;
|
||||
try {
|
||||
await runMigrations(writer);
|
||||
id = await insertVisit(writer, existing);
|
||||
} finally {
|
||||
await writer.destroy();
|
||||
}
|
||||
const result = run(
|
||||
'--config',
|
||||
configPath,
|
||||
'--visits-per-domain',
|
||||
'100',
|
||||
'--seed',
|
||||
'17',
|
||||
'--apply'
|
||||
);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.match(result.stdout, /Appended 300 synthetic visits/);
|
||||
const reader = await openDb(databasePath, { readOnly: true });
|
||||
try {
|
||||
const rows = await reader.selectFrom('visits').selectAll().execute();
|
||||
assert.equal(rows.length, 301);
|
||||
const original = rows.find((row) => row.id === id);
|
||||
assert.deepEqual({ ...original }, { id, ...existing });
|
||||
const synthetic = rows.filter((row) => row.user_agent?.endsWith(' GotoSeed/17'));
|
||||
for (const domain of Object.keys(fixture.domains)) {
|
||||
assert.equal(synthetic.filter((row) => row.domain === domain).length, 100);
|
||||
}
|
||||
} finally {
|
||||
await reader.destroy();
|
||||
}
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('a failed second batch rolls back its partial writes and preserves committed visits', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goto-seed-failure-'));
|
||||
try {
|
||||
const { configPath, databasePath } = await deployment(directory);
|
||||
const existing = {
|
||||
domain: 'one.example.test',
|
||||
path: '/keep',
|
||||
outcome: 'valid',
|
||||
visited_at: 1,
|
||||
ip: '198.51.100.1',
|
||||
user_agent: 'ExistingClient',
|
||||
accept_language: null,
|
||||
browser: 'Other',
|
||||
os: 'Other'
|
||||
};
|
||||
const writer = await openDb(databasePath);
|
||||
let id;
|
||||
try {
|
||||
await runMigrations(writer);
|
||||
id = await insertVisit(writer, existing);
|
||||
} finally {
|
||||
await writer.destroy();
|
||||
}
|
||||
const triggerDb = new DatabaseSync(databasePath);
|
||||
try {
|
||||
triggerDb.exec(`
|
||||
CREATE TRIGGER reject_synthetic_visit_275
|
||||
BEFORE INSERT ON visits
|
||||
WHEN NEW.user_agent LIKE '% GotoSeed/17'
|
||||
AND (SELECT COUNT(*) FROM visits WHERE user_agent LIKE '% GotoSeed/17') = 274
|
||||
BEGIN
|
||||
SELECT RAISE(ABORT, 'synthetic fixture failure');
|
||||
END
|
||||
`);
|
||||
} finally {
|
||||
triggerDb.close();
|
||||
}
|
||||
const result = run(
|
||||
'--config',
|
||||
configPath,
|
||||
'--visits-per-domain',
|
||||
'100',
|
||||
'--seed',
|
||||
'17',
|
||||
'--apply'
|
||||
);
|
||||
assert.equal(result.status, 1, result.stderr);
|
||||
assert.match(result.stderr, /synthetic fixture failure/);
|
||||
assert.match(result.stderr, /Stopped after 250 committed visits/);
|
||||
const reader = await openDb(databasePath, { readOnly: true });
|
||||
try {
|
||||
const rows = await reader.selectFrom('visits').selectAll().execute();
|
||||
const synthetic = rows.filter((row) => row.user_agent?.endsWith(' GotoSeed/17'));
|
||||
assert.equal(synthetic.length, 250);
|
||||
assert.equal(rows.length, 251);
|
||||
const original = rows.find((row) => row.id === id);
|
||||
assert.deepEqual({ ...original }, { id, ...existing });
|
||||
} finally {
|
||||
await reader.destroy();
|
||||
}
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('invalid counts and seeds fail before loading configuration', () => {
|
||||
const count = run('--config', '/missing/not-read.yaml', '--visits-per-domain', '0');
|
||||
assert.equal(count.status, 1);
|
||||
assert.match(count.stderr, /--visits-per-domain/);
|
||||
const seed = run('--config', '/missing/not-read.yaml', '--seed', '2147483647');
|
||||
assert.equal(seed.status, 1);
|
||||
assert.match(seed.stderr, /--seed must be between/);
|
||||
});
|
||||
Reference in new issue
Block a user