From e0008fe496948f59681782fb3640ac98333c8f1b Mon Sep 17 00:00:00 2001 From: eleith Date: Fri, 6 Nov 2026 16:25:24 -0800 Subject: [PATCH 1/4] fix(dev): allow SvelteKit generated type updates --- apps/web/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 24e1221..48131f3 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -20,10 +20,13 @@ COPY pnpm-lock.yaml pnpm-workspace.yaml ./ RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm fetch --frozen-lockfile COPY . . RUN --mount=type=cache,id=pnpm-store,target=/home/app/store pnpm install --offline --frozen-lockfile +# Only generated output and caches are writable; source binds stay read-only. RUN mkdir -p /m/apps/web/config /m/apps/web/data /m/packages/config/dist /m/packages/db/dist /m/apps/web/.svelte-kit \ + '/m/apps/web/node_modules/$app' \ /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp \ && chown -R ${LOCAL_UID}:${LOCAL_GID} /home/app /m/apps/web/data /m/packages/config/dist /m/packages/db/dist \ - /m/apps/web/.svelte-kit /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp + /m/apps/web/.svelte-kit '/m/apps/web/node_modules/$app' \ + /m/apps/web/node_modules/.vite /m/apps/web/node_modules/.vite-temp USER ${LOCAL_UID}:${LOCAL_GID} WORKDIR /m/apps/web EXPOSE 5173 -- 2.54.0 From 0492d7aab845ee32beb66cf915aba066f9520c71 Mon Sep 17 00:00:00 2001 From: eleith Date: Fri, 6 Nov 2026 16:25:36 -0800 Subject: [PATCH 2/4] feat(dev): add repeatable synthetic visit seeding --- .github/workflows/db.yml | 11 ++ .woodpecker/db.yml | 9 ++ package.json | 6 +- scripts/seed-visits-data.js | 153 +++++++++++++++++++++++ scripts/seed-visits.js | 169 +++++++++++++++++++++++++ scripts/seed-visits.test.js | 239 ++++++++++++++++++++++++++++++++++++ 6 files changed, 585 insertions(+), 2 deletions(-) create mode 100644 scripts/seed-visits-data.js create mode 100644 scripts/seed-visits.js create mode 100644 scripts/seed-visits.test.js diff --git a/.github/workflows/db.yml b/.github/workflows/db.yml index 7efd413..95cf9f3 100644 --- a/.github/workflows/db.yml +++ b/.github/workflows/db.yml @@ -9,6 +9,9 @@ on: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/config/** + - scripts/** + - tools/** pull_request: branches: [main] paths: @@ -17,6 +20,9 @@ on: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/config/** + - scripts/** + - tools/** permissions: contents: read @@ -48,3 +54,8 @@ jobs: run: pnpm --filter @goto/db lint - name: Unit tests run: pnpm --filter @goto/db test:coverage + - name: Development script checks + run: | + pnpm exec prettier --check scripts + pnpm exec eslint scripts + pnpm test:scripts diff --git a/.woodpecker/db.yml b/.woodpecker/db.yml index 547cab4..d338e62 100644 --- a/.woodpecker/db.yml +++ b/.woodpecker/db.yml @@ -12,6 +12,9 @@ when: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/config/** + - scripts/** + - tools/** - event: pull_request branch: main path: @@ -21,6 +24,9 @@ when: - .github/workflows/** - .woodpecker/** - packages/db/** + - packages/config/** + - scripts/** + - tools/** steps: prepare: @@ -52,6 +58,8 @@ steps: - corepack enable - pnpm --filter @goto/db check - pnpm --filter @goto/db lint + - pnpm exec prettier --check scripts + - pnpm exec eslint scripts test: image: node:26.5.0-bookworm @@ -61,3 +69,4 @@ steps: - npm install --global corepack@0.36.0 - corepack enable - pnpm --filter @goto/db test:coverage + - pnpm test:scripts diff --git a/package.json b/package.json index 68e4fd4..2991fd1 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "dev": "pnpm build:packages && pnpm --filter @goto/web dev", "dev:worker": "pnpm --filter @goto/worker dev", "cli": "pnpm build:packages && pnpm --filter @goto/cli build && node apps/cli/dist/index.js", + "seed:visits": "node --import ./tools/register.mjs scripts/seed-visits.js", "schema:generate": "pnpm --filter @goto/config schema:generate", "schema:check": "pnpm --filter @goto/config schema:check", "build:packages": "pnpm --filter './packages/*' -r build", @@ -18,8 +19,9 @@ "check": "pnpm -r check", "lint": "prettier --check . && eslint .", "format": "prettier --write .", - "test": "pnpm -r test", - "test:coverage": "pnpm -r test:coverage", + "test:scripts": "node --import ./tools/register.mjs --test scripts/*.test.js", + "test": "pnpm test:scripts && pnpm -r test", + "test:coverage": "pnpm test:scripts && pnpm -r test:coverage", "test:e2e": "pnpm --filter @goto/web test:e2e", "verify": "pnpm schema:check && pnpm build && pnpm check && pnpm lint && pnpm test:coverage && pnpm test:e2e && pnpm --filter @goto/worker test:smoke", "changeset": "changeset", diff --git a/scripts/seed-visits-data.js b/scripts/seed-visits-data.js new file mode 100644 index 0000000..7723b67 --- /dev/null +++ b/scripts/seed-visits-data.js @@ -0,0 +1,153 @@ +// Synthetic profiles use the existing approximate browser/OS family names. +const profiles = [ + { + weight: 36, + browser: 'Chrome', + os: 'Windows', + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/134.0 Safari/537.36' + }, + { + weight: 20, + browser: 'Chrome', + os: 'Android', + userAgent: 'Mozilla/5.0 (Linux; Android 15) Chrome/134.0 Mobile Safari/537.36' + }, + { + weight: 14, + browser: 'Safari', + os: 'macOS', + userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) Version/17.0 Safari/605.1.15' + }, + { + weight: 10, + browser: 'Safari', + os: 'iOS', + userAgent: + 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) Version/17.0 Mobile Safari/604.1' + }, + { + weight: 10, + browser: 'Firefox', + os: 'Linux', + userAgent: 'Mozilla/5.0 (X11; Linux x86_64) Firefox/136.0' + }, + { + weight: 5, + browser: 'Edge', + os: 'Windows', + userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/134.0 Safari/537.36 Edg/134.0' + }, + { + weight: 2, + browser: 'Opera', + os: 'Windows', + userAgent: 'Mozilla/5.0 (Windows NT 10.0) Chrome/134.0 Safari/537.36 OPR/118.0' + }, + { + weight: 2, + browser: 'Chrome', + os: 'ChromeOS', + userAgent: 'Mozilla/5.0 (X11; CrOS x86_64 16000.0.0) Chrome/134.0 Safari/537.36' + }, + { weight: 1, browser: 'Other', os: 'Other', userAgent: 'SyntheticClient/1.0' } +]; + +const languages = [ + 'en-US,en;q=0.9', + 'en-GB,en;q=0.9', + 'de-DE,de;q=0.9', + 'fr-FR,fr;q=0.9', + 'ja-JP,ja;q=0.9' +]; + +export function seedWindowStart(months, asOf) { + // Retention is UTC calendar months, including month-end clamping. + const date = new Date(asOf); + const day = date.getUTCDate(); + date.setUTCDate(1); + date.setUTCMonth(date.getUTCMonth() - months); + const lastDay = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() + 1, 0)).getUTCDate(); + date.setUTCDate(Math.min(day, lastDay)); + return date.getTime(); +} + +export function* generateVisits({ domains, visitsPerDomain, since, asOf, seed }) { + const random = seededRandom(seed); + for (const [domain, prepared] of Object.entries(domains)) { + const paths = requestPaths(prepared.links); + for (let index = 0; index < visitsPerDomain; index++) { + const visitedAt = visitTime(random, since, asOf); + const path = weightedChoice(random, paths).path; + const profile = weightedChoice(random, profiles); + const languageIndex = Math.floor(random() * languages.length); + yield { + domain, + path, + outcome: visitOutcome(prepared.links[path], visitedAt), + visited_at: visitedAt, + ip: '192.0.2.1', + user_agent: `${profile.userAgent} GotoSeed/${seed}`, + accept_language: languages[languageIndex], + browser: profile.browser, + os: profile.os + }; + } + } +} + +function requestPaths(links) { + let missingPath = '/__seed-missing__'; + while (Object.hasOwn(links, missingPath)) missingPath += '-'; + const paths = [ + { path: '/', weight: 10 }, + { path: missingPath, weight: 10 } + ]; + // Prefer configured links, with a separate share for base and unmatched requests. + const configured = Object.keys(links); + const popularity = configured.reduce((total, _path, index) => total + 1 / (index + 1), 0); + for (const [index, path] of configured.entries()) { + const weight = 80 / ((index + 1) * popularity); + paths.push({ path, weight }); + } + return paths; +} + +function visitOutcome(link, visitedAt) { + if (!link || !link.enabled) return 'invalid'; + if (link.valid_from !== undefined && visitedAt < Date.parse(link.valid_from)) return 'invalid'; + if (link.valid_until !== undefined && visitedAt >= Date.parse(link.valid_until)) return 'invalid'; + return 'valid'; +} + +function visitTime(random, since, asOf) { + while (true) { + const timestamp = since + Math.floor(random() * (asOf - since)); + const date = new Date(timestamp); + const weekday = date.getUTCDay(); + const hour = date.getUTCHours(); + const weekdayWeight = weekday === 0 || weekday === 6 ? 0.35 : 1; + const hourWeight = hour >= 8 && hour < 19 ? 1 : 0.3; + const recencyWeight = 0.4 + 0.6 * ((timestamp - since) / (asOf - since)); + const weight = weekdayWeight * hourWeight * recencyWeight; + if (random() < weight) return timestamp; + } +} + +function weightedChoice(random, entries) { + const total = entries.reduce((sum, entry) => sum + entry.weight, 0); + let remaining = random() * total; + for (const entry of entries) { + remaining -= entry.weight; + if (remaining < 0) return entry; + } + return entries.at(-1); +} + +function seededRandom(seed) { + // Park–Miller: repeatable fixture sampling, not cryptographic randomness. + let state = seed; + return function random() { + state = (state * 48271) % 2147483647; + return (state - 1) / 2147483646; + }; +} diff --git a/scripts/seed-visits.js b/scripts/seed-visits.js new file mode 100644 index 0000000..9751570 --- /dev/null +++ b/scripts/seed-visits.js @@ -0,0 +1,169 @@ +import { stat } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { setImmediate } from 'node:timers/promises'; +import { parseArgs } from 'node:util'; +import { loadConfigFile } from '../packages/config/src/index.ts'; +import { openDb, insertVisit } from '../packages/db/src/index.ts'; +import { generateVisits, seedWindowStart } from './seed-visits-data.js'; + +// Keep write transactions short while the development services share SQLite. +const batchSize = 250; +const help = `Development-only synthetic visit seeder (append-only). + +Usage: + pnpm run seed:visits --config [options] + +Options: + --config YAML configuration; required + --visits-per-domain Visits for each configured domain (default: 10000) + --seed Repeatable sampling seed, 1–2147483646 (default: 42) + --apply Append visits; otherwise preview without opening SQLite + --help Show this help + +Uses the configured visits database and retention window. The database must +already exist with the current visits schema; this tool never runs migrations. +Rerunning with --apply appends another dataset. Back up your development data +before applying. Synthetic rows use IP 192.0.2.1 and a GotoSeed/ UA marker. +`; + +async function main() { + const options = readOptions(); + if (options.help) { + console.log(help); + return; + } + const prepared = await loadConfigFile(options.configPath); + const plan = seedPlan(prepared, options); + printPlan(plan, options.apply); + if (!options.apply) return; + await appendVisits(plan); + console.log( + 'Prepared reports update at the next worker refresh; this tool does not trigger one.' + ); +} + +function readOptions() { + const { values } = parseArgs({ + options: { + config: { type: 'string' }, + 'visits-per-domain': { type: 'string', default: '10000' }, + seed: { type: 'string', default: '42' }, + apply: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false } + } + }); + if (values.help) return { help: true }; + if (!values.config?.trim()) throw new Error('--config is required; use --help for usage'); + const visitsPerDomain = positiveInteger(values['visits-per-domain'], '--visits-per-domain'); + const seed = positiveInteger(values.seed, '--seed'); + if (seed >= 2147483647) throw new Error('--seed must be between 1 and 2147483646'); + return { configPath: resolve(values.config), visitsPerDomain, seed, apply: values.apply }; +} + +function positiveInteger(value, name) { + const number = Number(value); + if (!/^[1-9]\d*$/.test(value) || !Number.isSafeInteger(number)) { + throw new Error(`${name} must be a positive safe integer`); + } + return number; +} + +function seedPlan(prepared, options) { + const { configuration, domains } = prepared; + const months = configuration.analytics.retention_months; + if (months === 0) throw new Error('Analytics collection is disabled; no visits will be seeded'); + if (Object.keys(domains).length === 0) throw new Error('No configured domains to seed'); + const total = Object.keys(domains).length * options.visitsPerDomain; + if (!Number.isSafeInteger(total)) + throw new Error('Total visit count exceeds the safe integer range'); + const databasePath = configuration.database.path; + if (databasePath === configuration.database.worker_path) { + throw new Error('Visits and worker database paths must differ'); + } + const asOf = Date.now(); + const since = seedWindowStart(months, asOf); + return { + databasePath, + domains, + since, + asOf, + visitsPerDomain: options.visitsPerDomain, + seed: options.seed, + total + }; +} + +function printPlan(plan, apply) { + console.log(apply ? 'Append synthetic visits' : 'Dry run — no database opened or changed'); + console.log(`Database: ${plan.databasePath}`); + console.log( + `UTC window: ${new Date(plan.since).toISOString()} to ${new Date(plan.asOf).toISOString()} (exclusive)` + ); + console.log(`Sampling seed: ${plan.seed} (timestamps are relative to this run)`); + for (const domain of Object.keys(plan.domains)) + console.log(` ${domain}: ${plan.visitsPerDomain} visits`); + console.log(`Total: ${plan.total} visits. Reruns append; existing data is not replaced.`); + if (!apply) + console.log('Review the target and back up your development data, then repeat with --apply.'); +} + +async function appendVisits(plan) { + const file = await stat(plan.databasePath).catch(() => { + throw new Error( + 'Cannot access the existing visits database; start the application normally before seeding' + ); + }); + if (!file.isFile()) throw new Error('Configured visits database is not a regular file'); + const db = await openDb(plan.databasePath); + let inserted = 0; + try { + // Validate insert columns before committing any rows, without changing schema. + const columns = [ + 'domain', + 'path', + 'outcome', + 'visited_at', + 'ip', + 'user_agent', + 'accept_language', + 'browser', + 'os' + ]; + await db.selectFrom('visits').select(columns).limit(0).execute(); + let batch = []; + for (const visit of generateVisits(plan)) { + batch.push(visit); + if (batch.length === batchSize) { + await appendBatch(db, batch); + inserted += batch.length; + batch = []; + await setImmediate(); + } + } + if (batch.length > 0) { + await appendBatch(db, batch); + inserted += batch.length; + } + console.log(`Appended ${inserted} synthetic visits.`); + } catch (error) { + console.error( + `Stopped after ${inserted} committed visits; completed batches remain in the database.` + ); + throw error; + } finally { + await db.destroy(); + } +} + +async function appendBatch(db, visits) { + await db.transaction().execute(async (transaction) => { + for (const visit of visits) await insertVisit(transaction, visit); + }); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/seed-visits.test.js b/scripts/seed-visits.test.js new file mode 100644 index 0000000..df10822 --- /dev/null +++ b/scripts/seed-visits.test.js @@ -0,0 +1,239 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { access, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { test } from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { validateConfig } from '../packages/config/src/index.ts'; +import { openDb, insertVisit, runMigrations } from '../packages/db/src/index.ts'; +import { classifyUserAgent } from '../packages/db/src/classify-user-agent.ts'; +import { generateVisits, seedWindowStart } from './seed-visits-data.js'; + +const script = fileURLToPath(new URL('./seed-visits.js', import.meta.url)); +const loader = fileURLToPath(new URL('../tools/register.mjs', import.meta.url)); +const fixture = { + site: { url: 'https://admin.example.test' }, + auth: { credentials: { username: 'test', password: 'seed-test-only-password' } }, + domains: { + 'one.example.test': { links: { '/docs': { url: 'https://example.org/docs' } } }, + 'two.example.test': { links: { '/docs': { url: 'https://example.com/docs' } } }, + 'three.example.test': { links: { '/docs': { url: 'https://example.net/docs' } } } + } +}; + +function run(...args) { + return spawnSync(process.execPath, ['--import', loader, script, ...args], { + encoding: 'utf8', + timeout: 15_000 + }); +} + +async function deployment(directory) { + await mkdir(join(directory, 'config')); + const configPath = join(directory, 'config/goto.yaml'); + await writeFile(configPath, JSON.stringify(fixture)); + return { configPath, databasePath: join(directory, 'data/goto.sqlite') }; +} + +test('calendar window clamps month ends and keeps UTC time', () => { + const since = seedWindowStart(3, Date.parse('2026-05-31T12:34:56.789Z')); + assert.equal(new Date(since).toISOString(), '2026-02-28T12:34:56.789Z'); + const leap = seedWindowStart(12, Date.parse('2024-02-29T12:00:00Z')); + assert.equal(new Date(leap).toISOString(), '2023-02-28T12:00:00.000Z'); +}); + +test('fixed inputs produce repeatable retained traffic with request-time outcomes and consistent profiles', () => { + const source = structuredClone(fixture); + source.domains['one.example.test'].links = { + '/': { url: 'https://example.org/' }, + '/docs': { + url: 'https://example.org/docs', + valid_from: '2026-03-31T00:00:00Z', + valid_until: '2026-04-30T00:00:00Z' + }, + '/disabled': { url: 'https://example.org/off', enabled: false }, + '/future': { url: 'https://example.org/future', valid_from: '2027-01-01T00:00:00Z' } + }; + const { domains } = validateConfig(source, { env: {} }); + const asOf = Date.parse('2026-05-31T12:00:00Z'); + const since = seedWindowStart(3, asOf); + const options = { domains, since, asOf, visitsPerDomain: 300, seed: 42 }; + const visits = [...generateVisits(options)]; + assert.deepEqual([...generateVisits(options)], visits); + assert.equal(visits.length, 900); + assert(visits.some((visit) => visit.path === '/' && visit.outcome === 'valid')); + assert(visits.some((visit) => visit.path === '/__seed-missing__' && visit.outcome === 'invalid')); + assert(visits.some((visit) => visit.path === '/disabled' && visit.outcome === 'invalid')); + const scheduledVisits = visits.filter( + (visit) => visit.domain === 'one.example.test' && visit.path === '/docs' + ); + const startsAt = Date.parse(source.domains['one.example.test'].links['/docs'].valid_from); + const endsAt = Date.parse(source.domains['one.example.test'].links['/docs'].valid_until); + assert(scheduledVisits.some((visit) => visit.visited_at < startsAt)); + assert( + scheduledVisits.some((visit) => visit.visited_at >= startsAt && visit.visited_at < endsAt) + ); + assert(scheduledVisits.some((visit) => visit.visited_at >= endsAt)); + for (const visit of scheduledVisits) { + const active = visit.visited_at >= startsAt && visit.visited_at < endsAt; + assert.equal(visit.outcome, active ? 'valid' : 'invalid'); + } + for (const visit of visits) { + assert(visit.visited_at >= since && visit.visited_at < asOf); + assert.equal(visit.ip, '192.0.2.1'); + assert(visit.user_agent.endsWith(' GotoSeed/42')); + assert.deepEqual(classifyUserAgent(visit.user_agent), { browser: visit.browser, os: visit.os }); + if (visit.path === '/future') assert.equal(visit.outcome, 'invalid'); + } +}); + +test('preview opens no SQLite file and applying refuses to create a missing database', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goto-seed-preview-')); + try { + const { configPath, databasePath } = await deployment(directory); + const preview = run('--config', configPath); + assert.equal(preview.status, 0, preview.stderr); + assert.match(preview.stdout, /Dry run/); + assert.match(preview.stdout, /Total: 30000 visits/); + await assert.rejects(access(databasePath), { code: 'ENOENT' }); + const apply = run('--config', configPath, '--apply'); + assert.equal(apply.status, 1, apply.stderr); + assert.match(apply.stderr, /existing visits database/); + await assert.rejects(access(databasePath), { code: 'ENOENT' }); + await writeFile(configPath, JSON.stringify({ ...fixture, analytics: { retention_months: 0 } })); + const disabled = run('--config', configPath, '--apply'); + assert.equal(disabled.status, 1); + assert.match(disabled.stderr, /Analytics collection is disabled/); + await assert.rejects(access(databasePath), { code: 'ENOENT' }); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('explicit apply appends all configured domains in bounded batches without replacing existing visits', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goto-seed-apply-')); + try { + const { configPath, databasePath } = await deployment(directory); + const existing = { + domain: 'one.example.test', + path: '/keep', + outcome: 'valid', + visited_at: 1, + ip: '198.51.100.1', + user_agent: 'ExistingClient', + accept_language: null, + browser: 'Other', + os: 'Other' + }; + const writer = await openDb(databasePath); + let id; + try { + await runMigrations(writer); + id = await insertVisit(writer, existing); + } finally { + await writer.destroy(); + } + const result = run( + '--config', + configPath, + '--visits-per-domain', + '100', + '--seed', + '17', + '--apply' + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Appended 300 synthetic visits/); + const reader = await openDb(databasePath, { readOnly: true }); + try { + const rows = await reader.selectFrom('visits').selectAll().execute(); + assert.equal(rows.length, 301); + const original = rows.find((row) => row.id === id); + assert.deepEqual({ ...original }, { id, ...existing }); + const synthetic = rows.filter((row) => row.user_agent?.endsWith(' GotoSeed/17')); + for (const domain of Object.keys(fixture.domains)) { + assert.equal(synthetic.filter((row) => row.domain === domain).length, 100); + } + } finally { + await reader.destroy(); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('a failed second batch rolls back its partial writes and preserves committed visits', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goto-seed-failure-')); + try { + const { configPath, databasePath } = await deployment(directory); + const existing = { + domain: 'one.example.test', + path: '/keep', + outcome: 'valid', + visited_at: 1, + ip: '198.51.100.1', + user_agent: 'ExistingClient', + accept_language: null, + browser: 'Other', + os: 'Other' + }; + const writer = await openDb(databasePath); + let id; + try { + await runMigrations(writer); + id = await insertVisit(writer, existing); + } finally { + await writer.destroy(); + } + const triggerDb = new DatabaseSync(databasePath); + try { + triggerDb.exec(` + CREATE TRIGGER reject_synthetic_visit_275 + BEFORE INSERT ON visits + WHEN NEW.user_agent LIKE '% GotoSeed/17' + AND (SELECT COUNT(*) FROM visits WHERE user_agent LIKE '% GotoSeed/17') = 274 + BEGIN + SELECT RAISE(ABORT, 'synthetic fixture failure'); + END + `); + } finally { + triggerDb.close(); + } + const result = run( + '--config', + configPath, + '--visits-per-domain', + '100', + '--seed', + '17', + '--apply' + ); + assert.equal(result.status, 1, result.stderr); + assert.match(result.stderr, /synthetic fixture failure/); + assert.match(result.stderr, /Stopped after 250 committed visits/); + const reader = await openDb(databasePath, { readOnly: true }); + try { + const rows = await reader.selectFrom('visits').selectAll().execute(); + const synthetic = rows.filter((row) => row.user_agent?.endsWith(' GotoSeed/17')); + assert.equal(synthetic.length, 250); + assert.equal(rows.length, 251); + const original = rows.find((row) => row.id === id); + assert.deepEqual({ ...original }, { id, ...existing }); + } finally { + await reader.destroy(); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('invalid counts and seeds fail before loading configuration', () => { + const count = run('--config', '/missing/not-read.yaml', '--visits-per-domain', '0'); + assert.equal(count.status, 1); + assert.match(count.stderr, /--visits-per-domain/); + const seed = run('--config', '/missing/not-read.yaml', '--seed', '2147483647'); + assert.equal(seed.status, 1); + assert.match(seed.stderr, /--seed must be between/); +}); -- 2.54.0 From 897ca1b375374786cdaad5c11902a5fbe12da8bf Mon Sep 17 00:00:00 2001 From: eleith Date: Fri, 6 Nov 2026 16:25:45 -0800 Subject: [PATCH 3/4] feat(analytics): prepare retained reports in the worker --- .../src/services/build-visit-reports.test.ts | 47 +++++ .../src/services/build-visit-reports.ts | 133 ++++++++++++++ .../worker/src/services/config-reload.test.ts | 43 +++-- apps/worker/src/services/config-reload.ts | 9 +- apps/worker/src/services/context.ts | 3 +- .../services/prepare-visit-reports.test.ts | 124 +++++++++++++ .../src/services/prepare-visit-reports.ts | 44 +++++ .../src/services/report-buckets.test.ts | 33 ++++ apps/worker/src/services/report-buckets.ts | 36 ++++ .../src/services/report-schedule.test.ts | 81 +++++++++ apps/worker/src/services/report-schedule.ts | 30 ++++ apps/worker/src/services/retention.test.ts | 115 +----------- apps/worker/src/services/retention.ts | 39 ----- apps/worker/src/services/startup.ts | 16 +- ....test.ts => refresh-visit-reports.test.ts} | 40 ++--- .../src/workflows/refresh-visit-reports.ts | 43 +++++ apps/worker/src/workflows/retain-visits.ts | 46 ----- apps/worker/src/workflows/specs.ts | 18 +- packages/db/package.json | 7 +- .../db/src/migrations/0006_visit_reports.ts | 14 ++ packages/db/src/migrations/index.ts | 4 +- packages/db/src/storage.test.ts | 9 +- packages/db/src/types.ts | 6 + packages/db/src/visit-reports/index.ts | 16 ++ packages/db/src/visit-reports/queries.test.ts | 151 ++++++++++++++++ packages/db/src/visit-reports/queries.ts | 87 ++++++++++ packages/db/src/visit-reports/storage.test.ts | 163 ++++++++++++++++++ packages/db/src/visit-reports/storage.ts | 59 +++++++ packages/db/src/visit-reports/types.test.ts | 20 +++ packages/db/src/visit-reports/types.ts | 76 ++++++++ 30 files changed, 1256 insertions(+), 256 deletions(-) create mode 100644 apps/worker/src/services/build-visit-reports.test.ts create mode 100644 apps/worker/src/services/build-visit-reports.ts create mode 100644 apps/worker/src/services/prepare-visit-reports.test.ts create mode 100644 apps/worker/src/services/prepare-visit-reports.ts create mode 100644 apps/worker/src/services/report-buckets.test.ts create mode 100644 apps/worker/src/services/report-buckets.ts create mode 100644 apps/worker/src/services/report-schedule.test.ts create mode 100644 apps/worker/src/services/report-schedule.ts rename apps/worker/src/workflows/{retain-visits.test.ts => refresh-visit-reports.test.ts} (55%) create mode 100644 apps/worker/src/workflows/refresh-visit-reports.ts delete mode 100644 apps/worker/src/workflows/retain-visits.ts create mode 100644 packages/db/src/migrations/0006_visit_reports.ts create mode 100644 packages/db/src/visit-reports/index.ts create mode 100644 packages/db/src/visit-reports/queries.test.ts create mode 100644 packages/db/src/visit-reports/queries.ts create mode 100644 packages/db/src/visit-reports/storage.test.ts create mode 100644 packages/db/src/visit-reports/storage.ts create mode 100644 packages/db/src/visit-reports/types.test.ts create mode 100644 packages/db/src/visit-reports/types.ts diff --git a/apps/worker/src/services/build-visit-reports.test.ts b/apps/worker/src/services/build-visit-reports.test.ts new file mode 100644 index 0000000..332a817 --- /dev/null +++ b/apps/worker/src/services/build-visit-reports.test.ts @@ -0,0 +1,47 @@ +import type { PreparedVisitReport } from '@goto/db/visit-reports'; +import { expect, test } from 'vitest'; +import { buildVisitReports } from './build-visit-reports.js'; +import { reportPeriod } from './report-buckets.js'; + +test('places counts on the correct UTC days, Monday weeks and calendar months', () => { + const period = reportPeriod(3, Date.parse('2026-07-01T12:00:00Z')); + const reports = buildVisitReports( + { + daily: [ + { scope: 'overview', day: '2026-05-31', total: 3, valid: 2, invalid: 1, base: 1 }, + { scope: 'overview', day: '2026-06-01', total: 5, valid: 1, invalid: 4, base: 2 } + ], + browsers: [], + operatingSystems: [] + }, + [], + period + ); + const report = JSON.parse( + reports.find((entry) => entry.key === 'overview')!.data + ) as PreparedVisitReport; + const firstDay = period.days.findIndex( + (bucket) => bucket.start === Date.parse('2026-05-30T00:00:00Z') + ); + const firstWeek = period.weeks.findIndex( + (bucket) => bucket.start === Date.parse('2026-05-25T00:00:00Z') + ); + + expect(report.daily.slice(firstDay, firstDay + 4)).toEqual([ + { total: 0, valid: 0, invalid: 0, base: 0 }, + { total: 3, valid: 2, invalid: 1, base: 1 }, + { total: 5, valid: 1, invalid: 4, base: 2 }, + { total: 0, valid: 0, invalid: 0, base: 0 } + ]); + expect(report.weekly.slice(firstWeek, firstWeek + 2)).toEqual([ + { total: 3, valid: 2, invalid: 1, base: 1 }, + { total: 5, valid: 1, invalid: 4, base: 2 } + ]); + expect(report.monthly).toEqual([ + { total: 0, valid: 0, invalid: 0, base: 0 }, + { total: 3, valid: 2, invalid: 1, base: 1 }, + { total: 5, valid: 1, invalid: 4, base: 2 }, + { total: 0, valid: 0, invalid: 0, base: 0 } + ]); + expect(report.totals).toEqual({ total: 8, valid: 3, invalid: 5, base: 3 }); +}); diff --git a/apps/worker/src/services/build-visit-reports.ts b/apps/worker/src/services/build-visit-reports.ts new file mode 100644 index 0000000..dcfe735 --- /dev/null +++ b/apps/worker/src/services/build-visit-reports.ts @@ -0,0 +1,133 @@ +import { + visitReportKey, + zeroVisitCounts, + type AggregatedVisitReports, + type DailyVisitCounts, + type ScopedFamilyCounts, + type PreparedVisitReport, + type ReportInventory, + type SerializedVisitReport, + type VisitCounts, + type VisitReportPeriod +} from '@goto/db/visit-reports'; + +interface ScopeCounts { + daily: DailyVisitCounts[]; + browsers: ScopedFamilyCounts[]; + operatingSystems: ScopedFamilyCounts[]; +} + +export function buildVisitReports( + aggregates: AggregatedVisitReports, + inventory: ReportInventory, + period: VisitReportPeriod +): SerializedVisitReport[] { + const indexed = indexCounts(aggregates); + const empty = emptyReport(period); + const reports: SerializedVisitReport[] = [{ key: 'empty', data: JSON.stringify(empty) }]; + const linkTotals = new Map(); + const domainTotals = new Map(); + + for (const entry of inventory) { + const paths = [...new Set([...entry.paths, '/'])]; + for (const path of paths) { + const key = visitReportKey(entry.domain, path); + const source = indexed.get(key); + if (!source) { + linkTotals.set(key, zeroVisitCounts()); + continue; + } + const report = scopeReport(source, period); + linkTotals.set(key, report.totals); + reports.push({ key, data: JSON.stringify(report) }); + } + const key = visitReportKey(entry.domain); + const source = indexed.get(key); + const report = source ? scopeReport(source, period) : { ...empty }; + report.linkCounts = paths.map((path) => { + const linkKey = visitReportKey(entry.domain, path); + return { path, counts: linkTotals.get(linkKey) ?? zeroVisitCounts() }; + }); + domainTotals.set(entry.domain, report.totals); + reports.push({ key, data: JSON.stringify(report) }); + } + const overviewSource = indexed.get('overview'); + const overview = overviewSource ? scopeReport(overviewSource, period) : { ...empty }; + overview.domainCounts = inventory.map((entry) => ({ + domain: entry.domain, + counts: domainTotals.get(entry.domain) ?? zeroVisitCounts() + })); + reports.push({ key: 'overview', data: JSON.stringify(overview) }); + return reports; +} + +function indexCounts(aggregates: AggregatedVisitReports): Map { + const indexed = new Map(); + function scope(key: string): ScopeCounts { + let counts = indexed.get(key); + if (!counts) { + counts = { daily: [], browsers: [], operatingSystems: [] }; + indexed.set(key, counts); + } + return counts; + } + for (const row of aggregates.daily) scope(row.scope).daily.push(row); + for (const row of aggregates.browsers) scope(row.scope).browsers.push(row); + for (const row of aggregates.operatingSystems) scope(row.scope).operatingSystems.push(row); + return indexed; +} + +function emptyReport(period: VisitReportPeriod): PreparedVisitReport { + return { + totals: zeroVisitCounts(), + daily: period.days.map(() => zeroVisitCounts()), + weekly: period.weeks.map(() => zeroVisitCounts()), + monthly: period.months.map(() => zeroVisitCounts()), + browsers: [], + operatingSystems: [], + domainCounts: [], + linkCounts: [] + }; +} + +function scopeReport(source: ScopeCounts, period: VisitReportPeriod): PreparedVisitReport { + const report = emptyReport(period); + const dayIndexes = new Map(); + period.days.forEach((bucket, index) => dayIndexes.set(bucket.start, index)); + for (const row of source.daily) { + const start = Date.parse(`${row.day}T00:00:00Z`); + const index = dayIndexes.get(start); + if (index === undefined) continue; + addCounts(report.daily[index]!, row); + addCounts(report.totals, row); + } + for (let index = 0; index < period.days.length; index++) { + const day = period.days[index]!; + const counts = report.daily[index]!; + const week = period.weeks.findIndex( + (bucket) => day.start >= bucket.start && day.start < bucket.end + ); + const month = period.months.findIndex( + (bucket) => day.start >= bucket.start && day.start < bucket.end + ); + if (week >= 0) addCounts(report.weekly[week]!, counts); + if (month >= 0) addCounts(report.monthly[month]!, counts); + } + report.browsers = source.browsers.map((row) => ({ name: row.name, counts: copyCounts(row) })); + report.operatingSystems = source.operatingSystems.map((row) => ({ + name: row.name, + counts: copyCounts(row) + })); + return report; +} + +function copyCounts(counts: VisitCounts): VisitCounts { + return { total: counts.total, valid: counts.valid, invalid: counts.invalid, base: counts.base }; +} + +function addCounts(target: VisitCounts, counts: VisitCounts): void { + target.total += counts.total; + target.valid += counts.valid; + target.invalid += counts.invalid; + target.base += counts.base; +} diff --git a/apps/worker/src/services/config-reload.test.ts b/apps/worker/src/services/config-reload.test.ts index d95a8af..e92fe4f 100644 --- a/apps/worker/src/services/config-reload.test.ts +++ b/apps/worker/src/services/config-reload.test.ts @@ -5,10 +5,10 @@ import { applyConfig, watchConfiguration } from './config-reload.js'; import type { WorkerContext } from './context.js'; import type { ShutdownResources } from './lifecycle.js'; import { logger } from './logger.js'; -import { requestRetention } from './retention.js'; +import { requestReportRefresh } from './report-schedule.js'; vi.mock('./logger.js', () => ({ logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } })); -vi.mock('./retention.js', () => ({ requestRetention: vi.fn() })); +vi.mock('./report-schedule.js', () => ({ requestReportRefresh: vi.fn() })); vi.mock('@goto/config', async (importOriginal) => ({ ...(await importOriginal()), startConfigReloading: vi.fn() @@ -29,14 +29,11 @@ function configuration( { env: {} } ); } - const engine = {} as OpenWorkflow; function context(): WorkerContext { - return { config: configuration().configuration, db: {} as WorkerContext['db'] }; + return { config: configuration().configuration, domains: {}, db: {} as WorkerContext['db'] }; } - beforeEach(() => vi.resetAllMocks()); - afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); @@ -57,7 +54,7 @@ test('requests restart for either database path when collection remains enabled' ).toBe(false); expect(restart).toHaveBeenCalledTimes(2); expect(ctx.config).toBe(accepted); - expect(requestRetention).not.toHaveBeenCalled(); + expect(requestReportRefresh).not.toHaveBeenCalled(); }); test.each([ @@ -74,43 +71,57 @@ test.each([ expect(await applyConfig(next, initial, restart, ctx, engine)).toBe(false); expect(ctx.config).toBe(initial.configuration); expect(restart).not.toHaveBeenCalled(); - expect(requestRetention).not.toHaveBeenCalled(); + expect(requestReportRefresh).not.toHaveBeenCalled(); expect(logger.warn).toHaveBeenCalledExactlyOnceWith( 'Analytics collection change requires coordinated restart; keeping current configuration' ); } ); -test('waits for a positive policy change to be pruned and accepts a later positive change', async () => { +test('waits for refreshed reports when retention changes', async () => { const initial = configuration(undefined, 9); const ctx = context(); ctx.config = initial.configuration; - const pruned = Promise.withResolvers(); - vi.mocked(requestRetention).mockReturnValueOnce(pruned.promise); + const refreshed = Promise.withResolvers(); + vi.mocked(requestReportRefresh).mockReturnValueOnce(refreshed.promise); const next = configuration(); const applying = applyConfig(next, initial, vi.fn(), ctx, engine); await Promise.resolve(); expect(ctx.config.analytics.retention_months).toBe(3); - expect(requestRetention).toHaveBeenCalledExactlyOnceWith(ctx, engine); + expect(requestReportRefresh).toHaveBeenCalledExactlyOnceWith(ctx, engine); let accepted = false; void applying.then(() => { accepted = true; }); await Promise.resolve(); expect(accepted).toBe(false); - pruned.resolve(); + refreshed.resolve(); expect(await applying).toBe(true); expect(await applyConfig(configuration(undefined, 12), next, vi.fn(), ctx, engine)).toBe(true); expect(ctx.config.analytics.retention_months).toBe(12); - expect(requestRetention).toHaveBeenCalledTimes(2); + expect(requestReportRefresh).toHaveBeenCalledTimes(2); }); -test('a failed positive policy reconciliation requests shutdown', async () => { +test('refreshes report inventory after an accepted domain/link change', async () => { + const initial = configuration(); + const next = configuration(); + next.domains['go.example.test'] = { + links: {}, + fallback: 404, + provenance: { fallback: 'builtin' } + }; + const ctx = context(); + expect(await applyConfig(next, initial, vi.fn(), ctx, engine)).toBe(true); + expect(ctx.domains).toBe(next.domains); + expect(requestReportRefresh).toHaveBeenCalledOnce(); +}); + +test('a failed refresh on positive policy reconciliation requests shutdown', async () => { const initial = configuration(undefined, 9); const ctx = context(); ctx.config = initial.configuration; const failure = new Error('Private storage detail'); - vi.mocked(requestRetention).mockRejectedValueOnce(failure); + vi.mocked(requestReportRefresh).mockRejectedValueOnce(failure); const stop = vi.fn(); await expect(applyConfig(configuration(), initial, stop, ctx, engine)).rejects.toBe(failure); expect(ctx.config.analytics.retention_months).toBe(3); diff --git a/apps/worker/src/services/config-reload.ts b/apps/worker/src/services/config-reload.ts index 339e3de..acdde68 100644 --- a/apps/worker/src/services/config-reload.ts +++ b/apps/worker/src/services/config-reload.ts @@ -1,9 +1,10 @@ import { startConfigReloading, type PreparedConfiguration } from '@goto/config'; +import { isDeepStrictEqual } from 'node:util'; import { logger } from './logger.js'; import type { ShutdownResources } from './lifecycle.js'; import type { OpenWorkflow } from 'openworkflow'; import type { WorkerContext } from './context.js'; -import { requestRetention } from './retention.js'; +import { requestReportRefresh } from './report-schedule.js'; interface ConfigurationSource { path: string; @@ -40,9 +41,11 @@ export async function applyConfig( } const currentMonths = context.config.analytics.retention_months; context.config = next.configuration; + context.domains = next.domains; try { - if (months !== previousMonths || months !== currentMonths) { - await requestRetention(context, engine); + const inventoryChanged = !isDeepStrictEqual(next.domains, previous.domains); + if (months !== previousMonths || months !== currentMonths || inventoryChanged) { + await requestReportRefresh(context, engine); } } catch (error) { logger.error('Unable to apply worker retention policy; stopping service'); diff --git a/apps/worker/src/services/context.ts b/apps/worker/src/services/context.ts index 6a79b32..34aed27 100644 --- a/apps/worker/src/services/context.ts +++ b/apps/worker/src/services/context.ts @@ -1,7 +1,8 @@ -import type { GotoConfiguration } from '@goto/config'; +import type { GotoConfiguration, PreparedConfiguration } from '@goto/config'; import type { openDb } from '@goto/db'; export interface WorkerContext { config: GotoConfiguration; + domains: PreparedConfiguration['domains']; db: Awaited>; } diff --git a/apps/worker/src/services/prepare-visit-reports.test.ts b/apps/worker/src/services/prepare-visit-reports.test.ts new file mode 100644 index 0000000..748da98 --- /dev/null +++ b/apps/worker/src/services/prepare-visit-reports.test.ts @@ -0,0 +1,124 @@ +import { validateConfig } from '@goto/config'; +import { openDb, runMigrations, insertVisit, pruneVisits, type ClassifiedVisit } from '@goto/db'; +import { readPreparedVisitReports } from '@goto/db/visit-reports'; +import { afterEach, expect, test, vi } from 'vitest'; +import { prepareVisitReports } from './prepare-visit-reports.js'; +import type { WorkerContext } from './context.js'; + +vi.mock('@goto/db', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, pruneVisits: vi.fn(actual.pruneVisits) }; +}); + +afterEach(() => vi.mocked(pruneVisits).mockReset()); + +async function context(): Promise { + const prepared = validateConfig( + { + auth: { credentials: { username: 'test', password: 'prepare-test-password' } }, + domains: { 'go.example.test': { links: { '/docs': { url: 'https://example.test/docs' } } } } + }, + { env: {} } + ); + const db = await openDb(':memory:'); + await runMigrations(db); + return { config: prepared.configuration, domains: prepared.domains, db }; +} +const visit: ClassifiedVisit = { + domain: 'go.example.test', + path: '/docs', + outcome: 'valid' as const, + visited_at: 1, + ip: '127.0.0.1', + user_agent: null, + accept_language: null, + browser: 'Firefox', + os: 'Linux' +}; + +function sum(values: { total: number }[]): number { + return values.reduce((total, value) => total + value.total, 0); +} + +test('one refresh publishes all scopes and chart resolutions, excludes/deletes expired data and is repeatable', async () => { + const ctx = await context(); + try { + const now = Date.parse('2026-05-31T12:00:00Z'); + const cutoff = Date.parse('2026-02-28T12:00:00Z'); + await ctx.db + .insertInto('visits') + .values(Array.from({ length: 1001 }, () => ({ ...visit, visited_at: cutoff - 1 }))) + .execute(); + await insertVisit(ctx.db, { ...visit, visited_at: cutoff }); + await insertVisit(ctx.db, { ...visit, path: '/', outcome: 'invalid', visited_at: now - 2000 }); + await prepareVisitReports(ctx, now); + const reports = await readPreparedVisitReports(ctx.db, 'go.example.test', '/docs'); + expect(reports?.report.totals).toEqual({ total: 2, valid: 1, invalid: 1, base: 1 }); + expect(reports?.selection.totals.total).toBe(1); + expect(sum(reports!.report.daily)).toBe(2); + expect(sum(reports!.report.weekly)).toBe(2); + expect(sum(reports!.report.monthly)).toBe(2); + expect(reports?.report.browsers).toEqual([ + { name: 'Firefox', counts: { total: 2, valid: 1, invalid: 1, base: 1 } } + ]); + expect(await ctx.db.selectFrom('visits').select('id').execute()).toHaveLength(2); + await prepareVisitReports(ctx, now); + expect((await readPreparedVisitReports(ctx.db))?.report.totals.total).toBe(2); + } finally { + await ctx.db.destroy(); + } +}); + +test('zero purges raw history and prepared output without leaving a longer-lived total', async () => { + const ctx = await context(); + try { + const now = Date.now(); + await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); + await prepareVisitReports(ctx, now); + ctx.config.analytics.retention_months = 0; + await prepareVisitReports(ctx, now); + expect(await ctx.db.selectFrom('visits').select('id').execute()).toEqual([]); + expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + } finally { + await ctx.db.destroy(); + } +}); + +test('clears old output before pruning and publishes nothing if pruning fails', async () => { + const ctx = await context(); + try { + const now = Date.parse('2026-05-31T12:00:00Z'); + await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); + await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + const failure = new Error('Pruning failed'); + vi.mocked(pruneVisits).mockImplementationOnce(async () => { + expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + throw failure; + }); + + await expect(prepareVisitReports(ctx, now)).rejects.toBe(failure); + expect(pruneVisits).toHaveBeenCalledExactlyOnceWith(ctx.db, Date.parse('2026-02-28T12:00:00Z')); + expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + } finally { + await ctx.db.destroy(); + } +}); + +test('stops pruning and does not publish obsolete output when positive retention changes', async () => { + const ctx = await context(); + try { + const now = Date.parse('2026-05-31T12:00:00Z'); + await insertVisit(ctx.db, { ...visit, visited_at: now - 1 }); + await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + vi.mocked(pruneVisits).mockImplementationOnce(async () => { + ctx.config.analytics.retention_months = 9; + return true; + }); + + await prepareVisitReports(ctx, now); + expect(pruneVisits).toHaveBeenCalledOnce(); + expect(await ctx.db.selectFrom('visit_reports').select('key').execute()).toEqual([]); + } finally { + await ctx.db.destroy(); + } +}); diff --git a/apps/worker/src/services/prepare-visit-reports.ts b/apps/worker/src/services/prepare-visit-reports.ts new file mode 100644 index 0000000..9e65462 --- /dev/null +++ b/apps/worker/src/services/prepare-visit-reports.ts @@ -0,0 +1,44 @@ +import { setImmediate } from 'node:timers/promises'; +import { pruneVisits, purgeVisits } from '@goto/db'; +import { + aggregateVisitReports, + clearVisitReports, + publishVisitReports, + type ReportInventory +} from '@goto/db/visit-reports'; +import type { WorkerContext } from './context.js'; +import { reportPeriod } from './report-buckets.js'; +import { buildVisitReports } from './build-visit-reports.js'; + +export async function prepareVisitReports( + context: WorkerContext, + asOf = Date.now() +): Promise { + const months = context.config.analytics.retention_months; + if (months === 0) { + await context.db.transaction().execute(async (transaction) => { + await clearVisitReports(transaction); + await purgeVisits(transaction); + }); + return; + } + + const inventory: ReportInventory = Object.entries(context.domains).map(([domain, prepared]) => ({ + domain, + paths: Object.keys(prepared.links) + })); + const period = reportPeriod(months, asOf); + const counts = await aggregateVisitReports(context.db, period.since, asOf, inventory); + const reports = buildVisitReports(counts, inventory, period); + if (context.config.analytics.retention_months !== months) return; + + // Retire old output before deleting any history represented by it. + await clearVisitReports(context.db); + while (context.config.analytics.retention_months === months) { + const more = await pruneVisits(context.db, period.since); + if (!more) break; + await setImmediate(); + } + if (context.config.analytics.retention_months !== months) return; + await publishVisitReports(context.db, period, reports); +} diff --git a/apps/worker/src/services/report-buckets.test.ts b/apps/worker/src/services/report-buckets.test.ts new file mode 100644 index 0000000..7d6932b --- /dev/null +++ b/apps/worker/src/services/report-buckets.test.ts @@ -0,0 +1,33 @@ +import { expect, test } from 'vitest'; +import { reportPeriod } from './report-buckets.js'; + +test('prepares UTC days, Monday weeks and calendar months with honest partial edges', () => { + const asOf = Date.parse('2026-05-31T12:34:56.789Z'); + const period = reportPeriod(3, asOf); + expect(period.since).toBe(Date.parse('2026-02-28T12:34:56.789Z')); + expect(period.asOf).toBe(asOf); + expect(period.days[0]).toEqual({ + start: Date.parse('2026-02-28T00:00:00Z'), + end: Date.parse('2026-03-01T00:00:00Z'), + partial: true + }); + expect(period.days[1]).toEqual({ + start: Date.parse('2026-03-01T00:00:00Z'), + end: Date.parse('2026-03-02T00:00:00Z'), + partial: false + }); + expect(period.weeks[1]).toEqual({ + start: Date.parse('2026-03-02T00:00:00Z'), + end: Date.parse('2026-03-09T00:00:00Z'), + partial: false + }); + expect(period.months[1]).toEqual({ + start: Date.parse('2026-03-01T00:00:00Z'), + end: Date.parse('2026-04-01T00:00:00Z'), + partial: false + }); + expect(new Date(period.weeks[0]!.start).getUTCDay()).toBe(1); + expect(period.months[0]!.start).toBe(Date.parse('2026-02-01T00:00:00Z')); + expect(period.days.at(-1)!.partial).toBe(true); + expect(period.months.at(-1)!.partial).toBe(true); +}); diff --git a/apps/worker/src/services/report-buckets.ts b/apps/worker/src/services/report-buckets.ts new file mode 100644 index 0000000..edfdbf2 --- /dev/null +++ b/apps/worker/src/services/report-buckets.ts @@ -0,0 +1,36 @@ +import type { TimeBucket, VisitReportPeriod } from '@goto/db/visit-reports'; +import { retentionCutoff, type RetentionMonths } from './retention.js'; + +type BucketSize = 'day' | 'week' | 'month'; + +export function reportPeriod(months: Exclude, asOf: number): VisitReportPeriod { + const since = retentionCutoff(months, asOf); + return { + timezone: 'UTC', + retentionMonths: months, + since, + asOf, + days: timeBuckets(since, asOf, 'day'), + weeks: timeBuckets(since, asOf, 'week'), + months: timeBuckets(since, asOf, 'month') + }; +} + +function timeBuckets(since: number, asOf: number, size: BucketSize): TimeBucket[] { + const date = new Date(since); + date.setUTCHours(0, 0, 0, 0); + if (size === 'week') { + const daysSinceMonday = (date.getUTCDay() + 6) % 7; + date.setUTCDate(date.getUTCDate() - daysSinceMonday); + } + if (size === 'month') date.setUTCDate(1); + const buckets: TimeBucket[] = []; + while (date.getTime() < asOf) { + const start = date.getTime(); + if (size === 'month') date.setUTCMonth(date.getUTCMonth() + 1); + else date.setUTCDate(date.getUTCDate() + (size === 'week' ? 7 : 1)); + const end = date.getTime(); + buckets.push({ start, end, partial: start < since || end > asOf }); + } + return buckets; +} diff --git a/apps/worker/src/services/report-schedule.test.ts b/apps/worker/src/services/report-schedule.test.ts new file mode 100644 index 0000000..25cdbdd --- /dev/null +++ b/apps/worker/src/services/report-schedule.test.ts @@ -0,0 +1,81 @@ +import { validateConfig } from '@goto/config'; +import { openDb, runMigrations } from '@goto/db'; +import type { OpenWorkflow } from 'openworkflow'; +import { expect, test, vi } from 'vitest'; +import type { WorkerContext } from './context.js'; +import { configureReportSchedule, requestReportRefresh } from './report-schedule.js'; +import { refreshVisitReports, reportSchedule } from '../workflows/specs.js'; + +async function context(): Promise { + const prepared = validateConfig( + { + auth: { credentials: { username: 'test', password: 'schedule-test-password' } }, + domains: {} + }, + { env: {} } + ); + const db = await openDb(':memory:'); + await runMigrations(db); + return { config: prepared.configuration, domains: prepared.domains, db }; +} +function client() { + const result = vi.fn(async () => 'completed'); + const engine = { + scheduleWorkflow: vi.fn(async () => {}), + schedules: { pause: vi.fn(async () => {}), resume: vi.fn(async () => {}) }, + runWorkflow: vi.fn(async () => ({ result })) + }; + return { engine, sdk: engine as unknown as OpenWorkflow, result }; +} + +test('zero pauses hourly scheduling and purges prepared output even if raw history is already empty', async () => { + const ctx = await context(); + const { engine, sdk } = client(); + try { + ctx.config.analytics.retention_months = 0; + await configureReportSchedule(ctx, sdk); + expect(engine.scheduleWorkflow).toHaveBeenCalledExactlyOnceWith( + refreshVisitReports, + reportSchedule.input, + reportSchedule + ); + expect(refreshVisitReports.name).toBe('retain-visits'); + expect(reportSchedule.id).toBe('goto:retain-visits'); + expect(reportSchedule.cron).toBe('0 * * * *'); + expect(reportSchedule.timezone).toBe('UTC'); + expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(reportSchedule.id); + await requestReportRefresh(ctx, sdk); + expect(engine.runWorkflow).not.toHaveBeenCalled(); + await ctx.db.insertInto('visit_reports').values({ key: 'old-output', data: '{}' }).execute(); + await requestReportRefresh(ctx, sdk); + expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(refreshVisitReports, {}); + } finally { + await ctx.db.destroy(); + } +}); + +test('positive policy resumes scheduling and startup waits for the preparation result', async () => { + const ctx = await context(); + const { engine, sdk, result } = client(); + try { + await configureReportSchedule(ctx, sdk); + expect(engine.schedules.resume).toHaveBeenCalledExactlyOnceWith(reportSchedule.id); + const completed = Promise.withResolvers(); + const requested = Promise.withResolvers(); + result.mockImplementationOnce(() => { + requested.resolve(); + return completed.promise; + }); + let finished = false; + const preparing = requestReportRefresh(ctx, sdk).then(() => { + finished = true; + }); + await requested.promise; + expect(finished).toBe(false); + completed.resolve('completed'); + await preparing; + expect(finished).toBe(true); + } finally { + await ctx.db.destroy(); + } +}); diff --git a/apps/worker/src/services/report-schedule.ts b/apps/worker/src/services/report-schedule.ts new file mode 100644 index 0000000..709c591 --- /dev/null +++ b/apps/worker/src/services/report-schedule.ts @@ -0,0 +1,30 @@ +import type { OpenWorkflow } from 'openworkflow'; +import { refreshVisitReports, reportSchedule } from '../workflows/specs.js'; +import type { WorkerContext } from './context.js'; + +export async function configureReportSchedule( + context: WorkerContext, + engine: OpenWorkflow +): Promise { + await engine.scheduleWorkflow(refreshVisitReports, reportSchedule.input, reportSchedule); + if (context.config.analytics.retention_months === 0) + await engine.schedules.pause(reportSchedule.id); + else await engine.schedules.resume(reportSchedule.id); +} + +export async function requestReportRefresh( + context: WorkerContext, + engine: OpenWorkflow +): Promise { + if (context.config.analytics.retention_months === 0) { + const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst(); + const output = await context.db + .selectFrom('visit_reports') + .select('key') + .limit(1) + .executeTakeFirst(); + if (!visit && !output) return; + } + const run = await engine.runWorkflow(refreshVisitReports, {}); + await run.result(); +} diff --git a/apps/worker/src/services/retention.test.ts b/apps/worker/src/services/retention.test.ts index 4c5d963..2a04701 100644 --- a/apps/worker/src/services/retention.test.ts +++ b/apps/worker/src/services/retention.test.ts @@ -1,15 +1,5 @@ -import { validateConfig } from '@goto/config'; -import { openDb, runMigrations, type ClassifiedVisit } from '@goto/db'; -import type { OpenWorkflow } from 'openworkflow'; -import { expect, test, vi } from 'vitest'; -import { - applyRetention, - configureRetentionSchedule, - requestRetention, - retentionCutoff -} from './retention.js'; -import type { WorkerContext } from './context.js'; -import { retainVisits, retentionSchedule } from '../workflows/specs.js'; +import { expect, test } from 'vitest'; +import { retentionCutoff } from './retention.js'; test('uses UTC calendar months, clamping the day while preserving the time', () => { const now = Date.parse('2026-05-31T12:34:56.789Z'); @@ -18,107 +8,6 @@ test('uses UTC calendar months, clamping the day while preserving the time', () expect(new Date(retentionCutoff(12, now)).toISOString()).toBe('2025-05-31T12:34:56.789Z'); }); -async function context(): Promise { - const db = await openDb(':memory:'); - await runMigrations(db); - const { configuration: config } = validateConfig( - { - auth: { credentials: { username: 'test', password: 'retention-test-password' } }, - domains: {} - }, - { env: {} } - ); - return { db, config }; -} - -function client() { - const result = vi.fn(async () => 'completed'); - const engine = { - scheduleWorkflow: vi.fn(async () => {}), - schedules: { pause: vi.fn(async () => {}), resume: vi.fn(async () => {}) }, - runWorkflow: vi.fn(async () => ({ result })) - }; - return { engine, result }; -} - -const visit: ClassifiedVisit = { - domain: 'go.example.test', - path: '/docs', - outcome: 'valid', - visited_at: 1, - ip: '127.0.0.1', - user_agent: null, - accept_language: null, - browser: 'Other', - os: 'Other' -}; - -test('zero pauses the native schedule and queues one purge only when data exists', async () => { - const ctx = await context(); - const { engine, result } = client(); - const sdk = engine as unknown as OpenWorkflow; - try { - ctx.config.analytics.retention_months = 0; - await configureRetentionSchedule(ctx, sdk); - expect(engine.scheduleWorkflow).toHaveBeenCalledExactlyOnceWith( - retainVisits, - retentionSchedule.input, - retentionSchedule - ); - expect(engine.schedules.pause).toHaveBeenCalledExactlyOnceWith(retentionSchedule.id); - expect(engine.schedules.resume).not.toHaveBeenCalled(); - await requestRetention(ctx, sdk); - expect(engine.runWorkflow).not.toHaveBeenCalled(); - await ctx.db.insertInto('visits').values(visit).execute(); - const completed = Promise.withResolvers(); - const requested = Promise.withResolvers(); - result.mockImplementationOnce(() => { - requested.resolve(); - return completed.promise; - }); - const requesting = requestRetention(ctx, sdk); - await requested.promise; - let finished = false; - void requesting.then(() => { - finished = true; - }); - await Promise.resolve(); - expect(finished).toBe(false); - completed.resolve('completed'); - await requesting; - expect(engine.runWorkflow).toHaveBeenCalledExactlyOnceWith(retainVisits, {}); - await applyRetention(ctx); - expect(await ctx.db.selectFrom('visits').selectAll().execute()).toEqual([]); - } finally { - await ctx.db.destroy(); - } -}); - -test('positive retention resumes the schedule and prunes every expired batch using one cutoff', async () => { - const ctx = await context(); - const { engine } = client(); - try { - const now = Date.parse('2026-05-31T12:34:56.789Z'); - const cutoff = retentionCutoff(3, now); - await ctx.db - .insertInto('visits') - .values([ - ...Array.from({ length: 1001 }, () => ({ ...visit, visited_at: cutoff - 1 })), - { ...visit, visited_at: cutoff } - ]) - .execute(); - await configureRetentionSchedule(ctx, engine as unknown as OpenWorkflow); - expect(engine.schedules.resume).toHaveBeenCalledExactlyOnceWith(retentionSchedule.id); - expect(engine.schedules.pause).not.toHaveBeenCalled(); - await applyRetention(ctx, now); - expect(await ctx.db.selectFrom('visits').select('visited_at').execute()).toEqual([ - { visited_at: cutoff } - ]); - } finally { - await ctx.db.destroy(); - } -}); - test('handles leap-day boundaries without overflowing into March', () => { expect(new Date(retentionCutoff(3, Date.parse('2024-05-31T01:00:00Z'))).toISOString()).toBe( '2024-02-29T01:00:00.000Z' diff --git a/apps/worker/src/services/retention.ts b/apps/worker/src/services/retention.ts index 210e659..4a52a19 100644 --- a/apps/worker/src/services/retention.ts +++ b/apps/worker/src/services/retention.ts @@ -1,46 +1,7 @@ -import { setImmediate } from 'node:timers/promises'; import type { GotoConfiguration } from '@goto/config'; -import { pruneVisits, purgeVisits } from '@goto/db'; -import type { OpenWorkflow } from 'openworkflow'; -import { retainVisits, retentionSchedule } from '../workflows/specs.js'; -import type { WorkerContext } from './context.js'; export type RetentionMonths = GotoConfiguration['analytics']['retention_months']; -export async function configureRetentionSchedule( - context: WorkerContext, - engine: OpenWorkflow -): Promise { - const enabled = context.config.analytics.retention_months !== 0; - await engine.scheduleWorkflow(retainVisits, retentionSchedule.input, retentionSchedule); - if (enabled) await engine.schedules.resume(retentionSchedule.id); - else await engine.schedules.pause(retentionSchedule.id); -} - -export async function requestRetention( - context: WorkerContext, - engine: OpenWorkflow -): Promise { - if (context.config.analytics.retention_months === 0) { - const visit = await context.db.selectFrom('visits').select('id').limit(1).executeTakeFirst(); - if (!visit) return; - } - const run = await engine.runWorkflow(retainVisits, {}); - await run.result(); -} - -export async function applyRetention(context: WorkerContext, now = Date.now()): Promise { - const months = context.config.analytics.retention_months; - if (months === 0) { - await purgeVisits(context.db); - return; - } - const before = retentionCutoff(months, now); - while (await pruneVisits(context.db, before)) { - await setImmediate(); - } -} - export function retentionCutoff(months: Exclude, now = Date.now()): number { const cutoff = new Date(now); const day = cutoff.getUTCDate(); diff --git a/apps/worker/src/services/startup.ts b/apps/worker/src/services/startup.ts index c70be28..6df148a 100644 --- a/apps/worker/src/services/startup.ts +++ b/apps/worker/src/services/startup.ts @@ -11,8 +11,8 @@ import { createShutdown, type ShutdownResources } from './lifecycle.js'; import { watchConfiguration } from './config-reload.js'; import { logger } from './logger.js'; import type { WorkerContext } from './context.js'; -import { configureRetentionSchedule, requestRetention } from './retention.js'; -import { registerRetentionWorkflow } from '../workflows/retain-visits.js'; +import { configureReportSchedule, requestReportRefresh } from './report-schedule.js'; +import { registerReportWorkflow } from '../workflows/refresh-visit-reports.js'; export interface WorkerService { port: number; @@ -44,7 +44,11 @@ export async function startWorkerService( const stop = createShutdown(resources); try { const db = await openVisitDatabase(visitDatabasePath, resources); - const context: WorkerContext = { config: prepared.configuration, db }; + const context: WorkerContext = { + config: prepared.configuration, + domains: prepared.domains, + db + }; const engine = await startWorkflowWorker(databasePath, context, resources); const host = captured.env.HOST ?? '0.0.0.0'; const healthPort = await startHealthServer(host, port, resources); @@ -79,12 +83,12 @@ async function startWorkflowWorker( resources.backend = backend; const engineLogger = createEngineLogger(logger); const engine = new OpenWorkflow({ backend, logger: engineLogger }); - registerRetentionWorkflow(engine, context); - await configureRetentionSchedule(context, engine); + registerReportWorkflow(engine, context); + await configureReportSchedule(context, engine); const workflowWorker = engine.newWorker({ concurrency: 1 }); resources.workflowWorker = workflowWorker; await workflowWorker.start(); - await requestRetention(context, engine); + await requestReportRefresh(context, engine); logger.info('Workflow worker started'); return engine; } diff --git a/apps/worker/src/workflows/retain-visits.test.ts b/apps/worker/src/workflows/refresh-visit-reports.test.ts similarity index 55% rename from apps/worker/src/workflows/retain-visits.test.ts rename to apps/worker/src/workflows/refresh-visit-reports.test.ts index 4863c3c..3acec86 100644 --- a/apps/worker/src/workflows/retain-visits.test.ts +++ b/apps/worker/src/workflows/refresh-visit-reports.test.ts @@ -1,30 +1,31 @@ import { validateConfig } from '@goto/config'; import { afterEach, expect, test, vi } from 'vitest'; import type { WorkerContext } from '../services/context.js'; -import { applyRetention } from '../services/retention.js'; -import { runRetentionWorkflow, type RetentionStep } from './retain-visits.js'; - -vi.mock('../services/retention.js', () => ({ applyRetention: vi.fn(async () => {}) })); +import { prepareVisitReports } from '../services/prepare-visit-reports.js'; +import { runReportRefresh, type ReportStep } from './refresh-visit-reports.js'; +vi.mock('../services/prepare-visit-reports.js', () => ({ + prepareVisitReports: vi.fn(async () => {}) +})); afterEach(() => vi.clearAllMocks()); function context(): WorkerContext { const prepared = validateConfig( - { - auth: { credentials: { username: 'test', password: 'jitter-test-password' } }, - domains: {} - }, + { auth: { credentials: { username: 'test', password: 'jitter-test-password' } }, domains: {} }, { env: {} } ); - return { config: prepared.configuration, db: {} as WorkerContext['db'] }; + return { + config: prepared.configuration, + domains: prepared.domains, + db: {} as WorkerContext['db'] + }; } - function steps() { const names: string[] = []; const sleep = vi .fn<(name: string, duration: string) => Promise>() .mockResolvedValue(undefined); - const step: RetentionStep = { + const step: ReportStep = { async run(config, fn) { names.push(config.name); return fn(); @@ -34,33 +35,32 @@ function steps() { return { step, names, sleep }; } -test('scheduled cleanup uses durable jitter; startup and policy runs do not', async () => { +test('hourly refresh uses durable jitter; startup/manual refreshes do not', async () => { const ctx = context(); const { step, names, sleep } = steps(); - await runRetentionWorkflow(ctx, { scheduled: true }, step); + await runReportRefresh(ctx, { scheduled: true }, step); expect(names).toEqual(['choose-jitter', 'retain-visits']); expect(sleep).toHaveBeenCalledOnce(); const [name, duration] = sleep.mock.calls[0]!; const seconds = Number(duration.slice(0, -1)); expect(name).toBe('retention-jitter'); - expect(duration.endsWith('s')).toBe(true); expect(seconds).toBeGreaterThanOrEqual(1); expect(seconds).toBeLessThanOrEqual(60); - expect(applyRetention).toHaveBeenCalledExactlyOnceWith(ctx); + expect(prepareVisitReports).toHaveBeenCalledExactlyOnceWith(ctx); vi.clearAllMocks(); names.length = 0; - await runRetentionWorkflow(ctx, {}, step); + await runReportRefresh(ctx, {}, step); expect(names).toEqual(['retain-visits']); expect(sleep).not.toHaveBeenCalled(); - expect(applyRetention).toHaveBeenCalledExactlyOnceWith(ctx); + expect(prepareVisitReports).toHaveBeenCalledExactlyOnceWith(ctx); }); -test('a prior scheduled run does no cleanup when the service starts with collection disabled', async () => { +test('old scheduled work does nothing when the service starts disabled', async () => { const ctx = context(); ctx.config.analytics.retention_months = 0; const { step, names, sleep } = steps(); - await runRetentionWorkflow(ctx, { scheduled: true }, step); + await runReportRefresh(ctx, { scheduled: true }, step); expect(names).toEqual([]); expect(sleep).not.toHaveBeenCalled(); - expect(applyRetention).not.toHaveBeenCalled(); + expect(prepareVisitReports).not.toHaveBeenCalled(); }); diff --git a/apps/worker/src/workflows/refresh-visit-reports.ts b/apps/worker/src/workflows/refresh-visit-reports.ts new file mode 100644 index 0000000..8e7312f --- /dev/null +++ b/apps/worker/src/workflows/refresh-visit-reports.ts @@ -0,0 +1,43 @@ +import { randomInt } from 'node:crypto'; +import type { OpenWorkflow } from 'openworkflow'; +import type { WorkerContext } from '../services/context.js'; +import { prepareVisitReports } from '../services/prepare-visit-reports.js'; +import { refreshVisitReports, type ReportRefreshInput } from './specs.js'; + +const MAX_JITTER_SECONDS = 60; + +export interface ReportStep { + run(config: { name: string }, fn: () => Promise): Promise; + sleep(name: string, duration: `${number}s`): Promise; +} + +export async function runReportRefresh( + context: WorkerContext, + input: ReportRefreshInput, + step: ReportStep +): Promise<'completed'> { + if (input.scheduled) { + if (context.config.analytics.retention_months === 0) return 'completed'; + const seconds = await step.run({ name: 'choose-jitter' }, chooseJitter); + const duration: `${number}s` = `${seconds}s`; + await step.sleep('retention-jitter', duration); + } + function prepare(): Promise { + return prepareVisitReports(context); + } + // Preserve the durable step identity used by already-queued maintenance runs. + await step.run({ name: 'retain-visits' }, prepare); + return 'completed'; +} + +export function registerReportWorkflow(engine: OpenWorkflow, context: WorkerContext): void { + function execute({ input, step }: { input: ReportRefreshInput; step: ReportStep }) { + return runReportRefresh(context, input, step); + } + engine.implementWorkflow(refreshVisitReports, execute); +} + +async function chooseJitter(): Promise { + const upperBound = MAX_JITTER_SECONDS + 1; + return randomInt(1, upperBound); +} diff --git a/apps/worker/src/workflows/retain-visits.ts b/apps/worker/src/workflows/retain-visits.ts deleted file mode 100644 index fb7668c..0000000 --- a/apps/worker/src/workflows/retain-visits.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { randomInt } from 'node:crypto'; -import type { OpenWorkflow } from 'openworkflow'; -import type { WorkerContext } from '../services/context.js'; -import { applyRetention } from '../services/retention.js'; -import { retainVisits, type RetainVisitsInput } from './specs.js'; - -const MAX_JITTER_SECONDS = 60; - -export interface RetentionStep { - run(config: { name: string }, fn: () => Promise): Promise; - sleep(name: string, duration: `${number}s`): Promise; -} - -function collectionDisabled(context: WorkerContext): boolean { - return context.config.analytics.retention_months === 0; -} - -async function chooseJitter(): Promise { - const upperBound = MAX_JITTER_SECONDS + 1; - return randomInt(1, upperBound); -} - -export async function runRetentionWorkflow( - context: WorkerContext, - input: RetainVisitsInput, - step: RetentionStep -): Promise<'completed'> { - if (input.scheduled) { - if (collectionDisabled(context)) return 'completed'; - const seconds = await step.run({ name: 'choose-jitter' }, chooseJitter); - const duration: `${number}s` = `${seconds}s`; - await step.sleep('retention-jitter', duration); - } - function applyPolicy(): Promise { - return applyRetention(context); - } - await step.run({ name: 'retain-visits' }, applyPolicy); - return 'completed'; -} - -export function registerRetentionWorkflow(engine: OpenWorkflow, context: WorkerContext): void { - function execute({ input, step }: { input: RetainVisitsInput; step: RetentionStep }) { - return runRetentionWorkflow(context, input, step); - } - engine.implementWorkflow(retainVisits, execute); -} diff --git a/apps/worker/src/workflows/specs.ts b/apps/worker/src/workflows/specs.ts index 29ceb9b..4d7d568 100644 --- a/apps/worker/src/workflows/specs.ts +++ b/apps/worker/src/workflows/specs.ts @@ -1,18 +1,22 @@ import { defineWorkflowSpec, type Workflow } from 'openworkflow'; -export interface RetainVisitsInput { +export interface ReportRefreshInput { scheduled?: boolean; } -export const retentionSchedule = { +export const reportSchedule = { id: 'goto:retain-visits', cron: '0 * * * *', timezone: 'UTC', input: { scheduled: true } }; -export const retainVisits: Workflow['spec'] = - defineWorkflowSpec({ - name: 'retain-visits', - schedules: [retentionSchedule] - }); +// Keep the durable identity so existing schedules and queued runs remain usable. +export const refreshVisitReports: Workflow< + ReportRefreshInput, + 'completed', + ReportRefreshInput +>['spec'] = defineWorkflowSpec({ + name: 'retain-visits', + schedules: [reportSchedule] +}); diff --git a/packages/db/package.json b/packages/db/package.json index b9afce7..47df02a 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -9,7 +9,12 @@ "development": "./src/index.ts", "import": "./dist/index.js" }, - "./visit-worker": "./dist/visit-worker.js" + "./visit-worker": "./dist/visit-worker.js", + "./visit-reports": { + "development": "./src/visit-reports/index.ts", + "types": "./dist/visit-reports/index.d.ts", + "import": "./dist/visit-reports/index.js" + } }, "files": [ "dist" diff --git a/packages/db/src/migrations/0006_visit_reports.ts b/packages/db/src/migrations/0006_visit_reports.ts new file mode 100644 index 0000000..d950ee0 --- /dev/null +++ b/packages/db/src/migrations/0006_visit_reports.ts @@ -0,0 +1,14 @@ +import type { Kysely, Migration } from 'kysely'; + +export const visitReports: Migration = { + async up(db: Kysely): Promise { + await db.schema + .createTable('visit_reports') + .addColumn('key', 'text', (column) => column.primaryKey()) + .addColumn('data', 'text', (column) => column.notNull()) + .execute(); + }, + async down(db: Kysely): Promise { + await db.schema.dropTable('visit_reports').execute(); + } +}; diff --git a/packages/db/src/migrations/index.ts b/packages/db/src/migrations/index.ts index 38062e3..218f0ed 100644 --- a/packages/db/src/migrations/index.ts +++ b/packages/db/src/migrations/index.ts @@ -3,11 +3,13 @@ import { addVisitClassification } from './0002_visit_classification.js'; import { visitRetention } from './0003_visit_retention.js'; import { removeVisitCollection } from './0004_remove_visit_collection.js'; import { addVisitOutcome } from './0005_visit_outcome.js'; +import { visitReports } from './0006_visit_reports.js'; export const migrations = { '0001_visits': initial, '0002_visit_classification': addVisitClassification, '0003_visit_retention': visitRetention, '0004_remove_visit_collection': removeVisitCollection, - '0005_visit_outcome': addVisitOutcome + '0005_visit_outcome': addVisitOutcome, + '0006_visit_reports': visitReports }; diff --git a/packages/db/src/storage.test.ts b/packages/db/src/storage.test.ts index bc8c155..8141ab3 100644 --- a/packages/db/src/storage.test.ts +++ b/packages/db/src/storage.test.ts @@ -54,7 +54,8 @@ test('opens a real WAL database and applies versioned migrations once', async () '0002_visit_classification', '0003_visit_retention', '0004_remove_visit_collection', - '0005_visit_outcome' + '0005_visit_outcome', + '0006_visit_reports' ]); const tables = await db.introspection.getTables(); const names = tables.map((table) => table.name); @@ -130,7 +131,8 @@ test('adds request-time labels without changing existing visit data', async () = '0002_visit_classification', '0003_visit_retention', '0004_remove_visit_collection', - '0005_visit_outcome' + '0005_visit_outcome', + '0006_visit_reports' ]); expect(await db.selectFrom('visits').selectAll().executeTakeFirstOrThrow()).toEqual({ ...visit, @@ -168,7 +170,8 @@ test('a failed migration rolls back schema and bookkeeping so startup can retry' '0002_visit_classification', '0003_visit_retention', '0004_remove_visit_collection', - '0005_visit_outcome' + '0005_visit_outcome', + '0006_visit_reports' ]); expect(await runMigrations(db)).toEqual([]); } finally { diff --git a/packages/db/src/types.ts b/packages/db/src/types.ts index 8fb3033..0af8fc5 100644 --- a/packages/db/src/types.ts +++ b/packages/db/src/types.ts @@ -15,8 +15,14 @@ export interface VisitTable { os: string; } +export interface VisitReportTable { + key: string; + data: string; +} + export interface Database { visits: VisitTable; + visit_reports: VisitReportTable; } export type ClassifiedVisit = Omit, 'id'>; diff --git a/packages/db/src/visit-reports/index.ts b/packages/db/src/visit-reports/index.ts new file mode 100644 index 0000000..5ee76d2 --- /dev/null +++ b/packages/db/src/visit-reports/index.ts @@ -0,0 +1,16 @@ +export { aggregateVisitReports } from './queries.js'; +export { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js'; +export { visitReportKey, zeroVisitCounts } from './types.js'; +export type { + AggregatedVisitReports, + DailyVisitCounts, + FamilyVisitCounts, + PreparedVisitReport, + ReportInventory, + ScopedFamilyCounts, + SerializedVisitReport, + StoredVisitReports, + TimeBucket, + VisitCounts, + VisitReportPeriod +} from './types.js'; diff --git a/packages/db/src/visit-reports/queries.test.ts b/packages/db/src/visit-reports/queries.test.ts new file mode 100644 index 0000000..b6f52fc --- /dev/null +++ b/packages/db/src/visit-reports/queries.test.ts @@ -0,0 +1,151 @@ +import { afterEach, beforeEach, expect, test } from 'vitest'; +import { openDb, runMigrations, type ClassifiedVisit } from '../index.js'; +import { aggregateVisitReports } from './queries.js'; +import { visitReportKey, type ReportInventory } from './types.js'; + +let db: Awaited>; + +const domain = 'go.example.test'; +const otherDomain = 'other.example.test'; +const since = Date.UTC(2026, 0, 31, 23, 59, 59, 999); +const nextDay = Date.UTC(2026, 1, 1); +const asOf = Date.UTC(2026, 1, 2); +const visit: ClassifiedVisit = { + domain, + path: '/docs', + outcome: 'valid', + visited_at: since, + ip: '127.0.0.1', + user_agent: null, + accept_language: null, + browser: 'Firefox', + os: 'Linux' +}; +const inventory: ReportInventory = [ + { domain, paths: ['/docs', '/new', '/docs'] }, + { domain: otherDomain, paths: ['/docs'] }, + { domain: 'empty.example.test', paths: [] } +]; + +beforeEach(async () => { + db = await openDb(':memory:'); + await runMigrations(db); +}); + +afterEach(async () => { + await db.destroy(); +}); + +async function seedVisits(): Promise { + await db + .insertInto('visits') + .values([ + visit, + { ...visit, path: '/', visited_at: nextDay, browser: 'Safari', os: 'macOS' }, + { ...visit, path: '/', outcome: 'invalid', visited_at: nextDay + 1 }, + { + ...visit, + path: '/unconfigured', + outcome: 'invalid', + visited_at: nextDay + 2, + browser: 'Other', + os: 'Other' + }, + { + ...visit, + domain: 'removed.example.test', + path: '/', + outcome: 'invalid', + visited_at: nextDay + 3 + }, + { + ...visit, + domain: otherDomain, + outcome: 'invalid', + visited_at: nextDay + 4, + os: 'Windows' + }, + { ...visit, visited_at: since - 1 }, + { ...visit, visited_at: asOf } + ]) + .execute(); +} + +test('aggregates UTC days and stored families for overview, current domains and known paths', async () => { + await seedVisits(); + const result = await aggregateVisitReports(db, since, asOf, inventory); + const overview = visitReportKey(); + const domainScope = visitReportKey(domain); + const docs = visitReportKey(domain, '/docs'); + const root = visitReportKey(domain, '/'); + const otherDomainScope = visitReportKey(otherDomain); + const otherDocs = visitReportKey(otherDomain, '/docs'); + + expect(result.daily).toHaveLength(8); + expect(result.daily).toEqual( + expect.arrayContaining([ + { scope: overview, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, + { scope: overview, day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 }, + { scope: domainScope, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, + { scope: domainScope, day: '2026-02-01', total: 3, valid: 1, invalid: 2, base: 2 }, + { scope: docs, day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, + { scope: root, day: '2026-02-01', total: 2, valid: 1, invalid: 1, base: 2 }, + { scope: otherDomainScope, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 }, + { scope: otherDocs, day: '2026-02-01', total: 1, valid: 0, invalid: 1, base: 0 } + ]) + ); + expect(result.browsers.filter(({ scope }) => scope === overview)).toEqual([ + { scope: overview, name: 'Firefox', total: 4, valid: 1, invalid: 3, base: 2 }, + { scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, + { scope: overview, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + ]); + expect(result.operatingSystems.filter(({ scope }) => scope === overview)).toEqual([ + { scope: overview, name: 'Linux', total: 3, valid: 1, invalid: 2, base: 2 }, + { scope: overview, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, + { scope: overview, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 }, + { scope: overview, name: 'macOS', total: 1, valid: 1, invalid: 0, base: 1 } + ]); + expect(result.browsers.filter(({ scope }) => scope === domainScope)).toEqual([ + { scope: domainScope, name: 'Firefox', total: 2, valid: 1, invalid: 1, base: 1 }, + { scope: domainScope, name: 'Other', total: 1, valid: 0, invalid: 1, base: 0 }, + { scope: domainScope, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + ]); + expect(result.browsers.filter(({ scope }) => scope === root)).toEqual([ + { scope: root, name: 'Firefox', total: 1, valid: 0, invalid: 1, base: 1 }, + { scope: root, name: 'Safari', total: 1, valid: 1, invalid: 0, base: 1 } + ]); + expect(result.operatingSystems.filter(({ scope }) => scope === otherDocs)).toEqual([ + { scope: otherDocs, name: 'Windows', total: 1, valid: 0, invalid: 1, base: 0 } + ]); + const familyScopes = new Set( + [...result.browsers, ...result.operatingSystems].map(({ scope }) => scope) + ); + expect(familyScopes).toEqual( + new Set([overview, domainScope, docs, root, otherDomainScope, otherDocs]) + ); +}); + +test('empty inventory still aggregates historical visits only under overview', async () => { + await seedVisits(); + const result = await aggregateVisitReports(db, since, asOf, []); + + expect(result.daily).toEqual([ + { scope: 'overview', day: '2026-01-31', total: 1, valid: 1, invalid: 0, base: 0 }, + { scope: 'overview', day: '2026-02-01', total: 5, valid: 1, invalid: 4, base: 3 } + ]); + expect(result.browsers.map(({ scope }) => scope)).toEqual(['overview', 'overview', 'overview']); + expect(result.operatingSystems.map(({ scope }) => scope)).toEqual([ + 'overview', + 'overview', + 'overview', + 'overview' + ]); +}); + +test('no visits produces no aggregate rows for configured scopes', async () => { + expect(await aggregateVisitReports(db, since, asOf, inventory)).toEqual({ + daily: [], + browsers: [], + operatingSystems: [] + }); +}); diff --git a/packages/db/src/visit-reports/queries.ts b/packages/db/src/visit-reports/queries.ts new file mode 100644 index 0000000..75be418 --- /dev/null +++ b/packages/db/src/visit-reports/queries.ts @@ -0,0 +1,87 @@ +import { sql, type Kysely } from 'kysely'; +import type { Database } from '../types.js'; +import { + visitReportKey, + type AggregatedVisitReports, + type DailyVisitCounts, + type ReportInventory, + type ScopedFamilyCounts +} from './types.js'; + +export async function aggregateVisitReports( + db: Kysely, + since: number, + asOf: number, + inventory: ReportInventory +): Promise { + const scopes = scopedVisits(since, asOf, inventory); + const counts = visitCountColumns(); + + return db.transaction().execute(async (transaction) => { + const daily = await sql` + ${scopes} + SELECT scope, date(visited_at / 1000.0, 'unixepoch') AS day, ${counts} + FROM scoped_visits + GROUP BY scope, day + ORDER BY scope, day + `.execute(transaction); + const browsers = await sql` + ${scopes} + SELECT scope, browser AS name, ${counts} + FROM scoped_visits + GROUP BY scope, browser + ORDER BY scope, name + `.execute(transaction); + const operatingSystems = await sql` + ${scopes} + SELECT scope, os AS name, ${counts} + FROM scoped_visits + GROUP BY scope, os + ORDER BY scope, name + `.execute(transaction); + + return { + daily: daily.rows, + browsers: browsers.rows, + operatingSystems: operatingSystems.rows + }; + }); +} + +function scopedVisits(since: number, asOf: number, inventory: ReportInventory) { + const domains = [...new Set(inventory.map(({ domain }) => domain))]; + const currentDomain = domains.length === 0 ? sql`0` : sql`domain IN (${sql.join(domains)})`; + const linkRules = inventory.map(({ domain, paths }) => { + const currentPaths = [...new Set([...paths, '/'])]; + return sql`(domain = ${domain} AND path IN (${sql.join(currentPaths)}))`; + }); + const currentLink = linkRules.length === 0 ? sql`0` : sql.join(linkRules, sql` OR `); + + return sql` + WITH retained_visits AS ( + SELECT domain, path, outcome, visited_at, browser, os + FROM visits + WHERE visited_at >= ${since} AND visited_at < ${asOf} + ), scoped_visits AS ( + SELECT ${visitReportKey()} AS scope, path, outcome, visited_at, browser, os + FROM retained_visits + UNION ALL + SELECT json_array(domain) AS scope, path, outcome, visited_at, browser, os + FROM retained_visits + WHERE ${currentDomain} + UNION ALL + SELECT json_array(domain, path) AS scope, path, outcome, visited_at, browser, os + FROM retained_visits + WHERE ${currentLink} + ) + `; +} + +function visitCountColumns() { + return sql` + COUNT(*) AS total, + SUM(CASE WHEN outcome = ${'valid'} THEN 1 ELSE 0 END) AS valid, + SUM(CASE WHEN outcome = ${'invalid'} THEN 1 ELSE 0 END) AS invalid, + SUM(CASE WHEN path = ${'/'} THEN 1 ELSE 0 END) AS base + `; +} diff --git a/packages/db/src/visit-reports/storage.test.ts b/packages/db/src/visit-reports/storage.test.ts new file mode 100644 index 0000000..e6b7f16 --- /dev/null +++ b/packages/db/src/visit-reports/storage.test.ts @@ -0,0 +1,163 @@ +import { afterEach, beforeEach, expect, test } from 'vitest'; +import { openDb, runMigrations } from '../index.js'; +import { clearVisitReports, publishVisitReports, readPreparedVisitReports } from './storage.js'; +import { + visitReportKey, + zeroVisitCounts, + type PreparedVisitReport, + type SerializedVisitReport, + type VisitCounts, + type VisitReportPeriod +} from './types.js'; + +let db: Awaited>; + +const domain = 'go.example.test'; +const period: VisitReportPeriod = { + timezone: 'UTC', + retentionMonths: 3, + since: Date.UTC(2026, 0, 1), + asOf: Date.UTC(2026, 0, 2), + days: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 0, 2), partial: false }], + weeks: [{ start: Date.UTC(2025, 11, 29), end: Date.UTC(2026, 0, 5), partial: true }], + months: [{ start: Date.UTC(2026, 0, 1), end: Date.UTC(2026, 1, 1), partial: true }] +}; + +beforeEach(async () => { + db = await openDb(':memory:'); + await runMigrations(db); +}); + +afterEach(async () => { + await db.destroy(); +}); + +function preparedReport(counts: VisitCounts = zeroVisitCounts()): PreparedVisitReport { + return { + totals: counts, + daily: [counts], + weekly: [counts], + monthly: [counts], + browsers: [], + operatingSystems: [], + domainCounts: [], + linkCounts: [] + }; +} + +function serializedReport(key: string, report: PreparedVisitReport): SerializedVisitReport { + return { key, data: JSON.stringify(report) }; +} + +test('reads the shared period, scope report and selected path from prepared rows', async () => { + const selection = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); + selection.browsers = [{ name: 'Firefox', counts: selection.totals }]; + selection.operatingSystems = [{ name: 'Linux', counts: selection.totals }]; + const domainReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 1 }); + domainReport.linkCounts = [{ path: '/docs', counts: selection.totals }]; + const overview = preparedReport({ total: 3, valid: 2, invalid: 1, base: 1 }); + overview.domainCounts = [{ domain, counts: domainReport.totals }]; + await publishVisitReports(db, period, [ + serializedReport('empty', preparedReport()), + serializedReport(visitReportKey(), overview), + serializedReport(visitReportKey(domain), domainReport), + serializedReport(visitReportKey(domain, '/docs'), selection) + ]); + + expect(await readPreparedVisitReports(db)).toEqual({ + period, + report: overview, + selection: overview + }); + expect(await readPreparedVisitReports(db, domain)).toEqual({ + period, + report: domainReport, + selection: domainReport + }); + expect(await readPreparedVisitReports(db, domain, '/docs')).toEqual({ + period, + report: domainReport, + selection + }); +}); + +test('publication replaces all output rows and a failed replacement preserves the previous set', async () => { + const oldReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); + await publishVisitReports(db, period, [ + serializedReport(visitReportKey(), oldReport), + serializedReport(visitReportKey(domain), oldReport) + ]); + const replacementPeriod = { ...period, retentionMonths: 9 as const }; + const replacementReport = preparedReport({ total: 2, valid: 1, invalid: 1, base: 0 }); + const replacementRows = [serializedReport(visitReportKey(), replacementReport)]; + await publishVisitReports(db, replacementPeriod, replacementRows); + + expect(await db.selectFrom('visit_reports').select('key').orderBy('key').execute()).toEqual([ + { key: 'overview' }, + { key: 'period' } + ]); + expect(await readPreparedVisitReports(db, domain)).toBeNull(); + await expect( + publishVisitReports(db, period, [ + serializedReport(visitReportKey(), oldReport), + serializedReport(visitReportKey(), oldReport) + ]) + ).rejects.toThrow(); + expect(await readPreparedVisitReports(db)).toEqual({ + period: replacementPeriod, + report: replacementReport, + selection: replacementReport + }); +}); + +test('a published zero-count path uses the empty template but a missing domain stays unavailable', async () => { + const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 1 }); + domainReport.linkCounts = [{ path: '/new', counts: zeroVisitCounts() }]; + const empty = preparedReport(); + await publishVisitReports(db, period, [ + serializedReport(visitReportKey(domain), domainReport), + serializedReport('empty', empty) + ]); + + expect(await readPreparedVisitReports(db, domain, '/new')).toEqual({ + period, + report: domainReport, + selection: empty + }); + expect(await readPreparedVisitReports(db, 'new.example.test', '/new')).toBeNull(); +}); + +test('missing link bodies stay unavailable for unpublished paths and populated links', async () => { + const domainReport = preparedReport({ total: 1, valid: 1, invalid: 0, base: 0 }); + domainReport.linkCounts = [{ path: '/docs', counts: domainReport.totals }]; + await publishVisitReports(db, period, [ + serializedReport(visitReportKey(domain), domainReport), + serializedReport('empty', preparedReport()) + ]); + + expect(await readPreparedVisitReports(db, domain, '/new')).toBeNull(); + expect(await readPreparedVisitReports(db, domain, '/docs')).toBeNull(); +}); + +test('missing metadata is unavailable and clearing deletes every output row', async () => { + expect(await readPreparedVisitReports(db)).toBeNull(); + await db + .insertInto('visit_reports') + .values(serializedReport(visitReportKey(), preparedReport())) + .execute(); + expect(await readPreparedVisitReports(db)).toBeNull(); + + await publishVisitReports(db, period, [ + serializedReport(visitReportKey(), preparedReport()), + serializedReport('empty', preparedReport()) + ]); + await clearVisitReports(db); + expect(await db.selectFrom('visit_reports').selectAll().execute()).toEqual([]); + expect(await readPreparedVisitReports(db)).toBeNull(); +}); + +test('malformed stored JSON is left for the caller to handle', async () => { + await publishVisitReports(db, period, [{ key: visitReportKey(), data: '{' }]); + + await expect(readPreparedVisitReports(db)).rejects.toThrow(SyntaxError); +}); diff --git a/packages/db/src/visit-reports/storage.ts b/packages/db/src/visit-reports/storage.ts new file mode 100644 index 0000000..0f0b85b --- /dev/null +++ b/packages/db/src/visit-reports/storage.ts @@ -0,0 +1,59 @@ +import type { Kysely } from 'kysely'; +import type { Database } from '../types.js'; +import { + visitReportKey, + type PreparedVisitReport, + type SerializedVisitReport, + type StoredVisitReports, + type VisitReportPeriod +} from './types.js'; + +export async function clearVisitReports(db: Kysely): Promise { + await db.deleteFrom('visit_reports').execute(); +} + +export async function publishVisitReports( + db: Kysely, + period: VisitReportPeriod, + reports: SerializedVisitReport[] +): Promise { + const rows = [{ key: 'period', data: JSON.stringify(period) }, ...reports]; + + await db.transaction().execute(async (transaction) => { + await transaction.deleteFrom('visit_reports').execute(); + await transaction.insertInto('visit_reports').values(rows).execute(); + }); +} + +export async function readPreparedVisitReports( + db: Kysely, + domain?: string, + path?: string +): Promise { + const reportKey = visitReportKey(domain); + const selectionKey = visitReportKey(domain, path); + const keys = [...new Set(['period', reportKey, selectionKey, 'empty'])]; + const rows = await db.selectFrom('visit_reports').selectAll().where('key', 'in', keys).execute(); + const dataByKey = new Map(rows.map(({ key, data }) => [key, data])); + const periodData = dataByKey.get('period'); + const reportData = dataByKey.get(reportKey); + if (periodData === undefined || reportData === undefined) return null; + + const report = JSON.parse(reportData) as PreparedVisitReport; + let selection = report; + if (selectionKey !== reportKey) { + let selectionData = dataByKey.get(selectionKey); + if (selectionData === undefined) { + const linkCounts = report.linkCounts.find((link) => link.path === path)?.counts; + if (linkCounts?.total !== 0) return null; + selectionData = dataByKey.get('empty'); + } + if (selectionData === undefined) return null; + selection = JSON.parse(selectionData) as PreparedVisitReport; + } + return { + period: JSON.parse(periodData) as VisitReportPeriod, + report, + selection + }; +} diff --git a/packages/db/src/visit-reports/types.test.ts b/packages/db/src/visit-reports/types.test.ts new file mode 100644 index 0000000..eaa3634 --- /dev/null +++ b/packages/db/src/visit-reports/types.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from 'vitest'; +import { visitReportKey, zeroVisitCounts } from './types.js'; + +test('zero counts are fresh objects', () => { + const first = zeroVisitCounts(); + const second = zeroVisitCounts(); + first.total = 1; + + expect(second).toEqual({ total: 0, valid: 0, invalid: 0, base: 0 }); + expect(first).not.toBe(second); +}); + +test('scope keys use overview or JSON-encoded domains and exact paths', () => { + expect(visitReportKey()).toBe('overview'); + expect(visitReportKey('go.example.test')).toBe('["go.example.test"]'); + expect(visitReportKey('go.example.test', '/')).toBe('["go.example.test","/"]'); + expect(visitReportKey('go.example.test', '/café,"docs')).toBe( + JSON.stringify(['go.example.test', '/café,"docs']) + ); +}); diff --git a/packages/db/src/visit-reports/types.ts b/packages/db/src/visit-reports/types.ts new file mode 100644 index 0000000..68c9abe --- /dev/null +++ b/packages/db/src/visit-reports/types.ts @@ -0,0 +1,76 @@ +export interface VisitCounts { + total: number; + valid: number; + invalid: number; + base: number; +} + +export function zeroVisitCounts(): VisitCounts { + return { total: 0, valid: 0, invalid: 0, base: 0 }; +} + +export interface TimeBucket { + start: number; + end: number; + partial: boolean; +} + +export interface VisitReportPeriod { + timezone: 'UTC'; + retentionMonths: 3 | 9 | 12; + since: number; + asOf: number; + days: TimeBucket[]; + weeks: TimeBucket[]; + months: TimeBucket[]; +} + +export interface FamilyVisitCounts { + name: string; + counts: VisitCounts; +} + +export interface PreparedVisitReport { + totals: VisitCounts; + daily: VisitCounts[]; + weekly: VisitCounts[]; + monthly: VisitCounts[]; + browsers: FamilyVisitCounts[]; + operatingSystems: FamilyVisitCounts[]; + domainCounts: Array<{ domain: string; counts: VisitCounts }>; + linkCounts: Array<{ path: string; counts: VisitCounts }>; +} + +export type ReportInventory = Array<{ domain: string; paths: string[] }>; + +export function visitReportKey(domain?: string, path?: string): string { + if (domain === undefined) return 'overview'; + return JSON.stringify(path === undefined ? [domain] : [domain, path]); +} + +export interface SerializedVisitReport { + key: string; + data: string; +} + +export interface StoredVisitReports { + period: VisitReportPeriod; + report: PreparedVisitReport; + selection: PreparedVisitReport; +} + +export interface DailyVisitCounts extends VisitCounts { + scope: string; + day: string; +} + +export interface ScopedFamilyCounts extends VisitCounts { + scope: string; + name: string; +} + +export interface AggregatedVisitReports { + daily: DailyVisitCounts[]; + browsers: ScopedFamilyCounts[]; + operatingSystems: ScopedFamilyCounts[]; +} -- 2.54.0 From 80d5b04e23fc5addbc6c96058dd72083f1c5e6e4 Mon Sep 17 00:00:00 2001 From: eleith Date: Fri, 6 Nov 2026 16:26:03 -0800 Subject: [PATCH 4/4] feat(admin): add prepared overview and link dashboards --- .changeset/admin-overview.md | 7 + .github/workflows/web.yml | 2 + .woodpecker/web.yml | 2 + apps/web/e2e/auth.e2e.ts | 80 ++++++++- apps/web/e2e/server.mjs | 106 +++++++++--- apps/web/e2e/support/test.ts | 12 +- apps/web/package.json | 2 +- .../components/{ => admin}/AdminNav.svelte | 5 +- .../src/lib/components/admin/AdminPage.svelte | 39 +++++ .../lib/components/admin/ReportNotice.svelte | 10 ++ .../src/lib/components/admin/StatCard.svelte | 9 + .../src/lib/components/admin/format.test.ts | 13 ++ apps/web/src/lib/components/admin/format.ts | 17 ++ apps/web/src/lib/server/link-state.ts | 9 + apps/web/src/lib/server/visit-reports.test.ts | 91 ++++++++++ apps/web/src/lib/server/visit-reports.ts | 35 ++++ apps/web/src/lib/styles/admin.css | 155 ++++++++++++++++++ .../(admin)/(auth)/admin/+page.server.ts | 39 +++-- .../routes/(admin)/(auth)/admin/+page.svelte | 155 ++++-------------- .../admin/domains/[domain]/+page.server.ts | 64 ++++++++ .../admin/domains/[domain]/+page.svelte | 71 ++++++++ .../[domain]/links/[...path]/+page.server.ts | 36 ++++ .../[domain]/links/[...path]/+page.svelte | 33 ++++ .../links/[...path]/page.server.test.ts | 105 ++++++++++++ .../domains/[domain]/page.server.test.ts | 121 ++++++++++++++ .../(admin)/(auth)/admin/page.server.test.ts | 76 +++++++++ 26 files changed, 1127 insertions(+), 167 deletions(-) create mode 100644 .changeset/admin-overview.md rename apps/web/src/lib/components/{ => admin}/AdminNav.svelte (94%) create mode 100644 apps/web/src/lib/components/admin/AdminPage.svelte create mode 100644 apps/web/src/lib/components/admin/ReportNotice.svelte create mode 100644 apps/web/src/lib/components/admin/StatCard.svelte create mode 100644 apps/web/src/lib/components/admin/format.test.ts create mode 100644 apps/web/src/lib/components/admin/format.ts create mode 100644 apps/web/src/lib/server/link-state.ts create mode 100644 apps/web/src/lib/server/visit-reports.test.ts create mode 100644 apps/web/src/lib/server/visit-reports.ts create mode 100644 apps/web/src/lib/styles/admin.css create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts create mode 100644 apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts diff --git a/.changeset/admin-overview.md b/.changeset/admin-overview.md new file mode 100644 index 0000000..566f1f1 --- /dev/null +++ b/.changeset/admin-overview.md @@ -0,0 +1,7 @@ +--- +'@goto/web': minor +'@goto/worker': minor +'@goto/cli': minor +--- + +Add an authenticated admin overview, domain summaries and dedicated link pages with current inventory and retained visits. One hourly background job, with scheduled jitter, prepares all reports in SQLite and coordinates retention cleanup. Admin pages read prepared snapshots directly and display their update time; no reporting API, token or worker URL is required. The job also prepares UTC daily, weekly and monthly series and browser/OS family counts for later visualization. Charts and the Health page are not included. diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 5911284..5e624b4 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -11,6 +11,7 @@ on: - .github/workflows/** - .woodpecker/** - apps/web/** + - apps/worker/** - packages/** pull_request: branches: [main] @@ -21,6 +22,7 @@ on: - .github/workflows/** - .woodpecker/** - apps/web/** + - apps/worker/** - packages/** permissions: diff --git a/.woodpecker/web.yml b/.woodpecker/web.yml index 169eaac..c87a6b4 100644 --- a/.woodpecker/web.yml +++ b/.woodpecker/web.yml @@ -13,6 +13,7 @@ when: - .github/workflows/** - .woodpecker/** - apps/web/** + - apps/worker/** - packages/** - event: pull_request path: @@ -23,6 +24,7 @@ when: - .github/workflows/** - .woodpecker/** - apps/web/** + - apps/worker/** - packages/** - event: tag ref: 'refs/tags/@goto/web@*' diff --git a/apps/web/e2e/auth.e2e.ts b/apps/web/e2e/auth.e2e.ts index 3c914a7..e4f7001 100644 --- a/apps/web/e2e/auth.e2e.ts +++ b/apps/web/e2e/auth.e2e.ts @@ -1,6 +1,8 @@ import { expect, test } from './support/test.ts'; -test('shared credentials protect the read-only inventory and sign-out removes access', async ({ +test.use({ withWorker: true }); + +test('shared credentials protect overview, domain and link pages and sign-out removes access', async ({ page, appServer }) => { @@ -13,16 +15,82 @@ test('shared credentials protect the read-only inventory and sign-out removes ac await page.getByLabel('Password', { exact: true }).fill('e2e-only-password'); await page.getByRole('button', { name: 'Sign in', exact: true }).click(); - await expect(page.getByRole('heading', { name: 'Configured links', exact: true })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Overview', exact: true })).toBeVisible(); + const visitsValue = page + .getByRole('region', { name: 'Valid visits', exact: true }) + .locator('.admin-stat-value'); + await expect(page.getByRole('region', { name: 'Domains', exact: true })).toContainText('2'); + await expect(visitsValue).toHaveText('3'); + const retentionValue = page + .getByRole('region', { name: 'Retention', exact: true }) + .locator('.admin-stat-value'); + await expect(retentionValue).toHaveText('3 months'); + await expect(page.locator('.admin-cards').getByRole('region')).toHaveCount(4); + const domainLink = page.getByRole('link', { name: 'go.example.org', exact: true }); + const domainRow = page.getByRole('row').filter({ has: domainLink }); + await expect(domainRow.getByRole('cell')).toHaveText(['6', '2']); + await domainLink.click(); + await expect(page.getByRole('heading', { name: 'go.example.org', exact: true })).toBeVisible(); + await expect(visitsValue).toHaveText('2'); + await expect(page.locator('.admin-cards .admin-stat-label')).toHaveText([ + 'Links', + 'Valid visits', + 'Invalid clicks', + 'Retention' + ]); await expect( - page - .getByRole('region', { name: 'Links on go.example.org', exact: true }) - .getByRole('link', { name: 'https://example.org/first-domain', exact: true }) + page.getByRole('region', { name: 'Invalid clicks', exact: true }).locator('.admin-stat-value') + ).toHaveText('1'); + const linksTable = page.getByRole('region', { name: 'Links on go.example.org', exact: true }); + const baseTable = page.getByRole('region', { name: 'Base URL on go.example.org', exact: true }); + const columns = ['Path', 'Destination', 'State', 'Visits']; + await expect(linksTable.getByRole('columnheader')).toHaveText(columns); + const historicalInvalid = linksTable + .getByRole('row') + .filter({ has: page.getByRole('link', { name: '/a/b', exact: true }) }); + await expect(historicalInvalid.getByRole('cell').last()).toHaveText('0'); + await expect(baseTable.getByRole('columnheader')).toHaveText(columns); + await expect(baseTable.getByRole('cell')).toHaveText(['404 Not found', 'Fallback', '0']); + await expect( + linksTable.getByRole('link', { name: 'https://example.org/first-domain', exact: true }) ).toBeVisible(); + const domainBreadcrumb = page + .getByRole('navigation', { name: 'Breadcrumb', exact: true }) + .getByRole('link', { name: 'go.example.org', exact: true }); + const docsPath = '/admin/domains/go.example.org/links/docs'; + await page.getByRole('link', { name: '/docs', exact: true }).click(); + await expect(page).toHaveURL(`${appServer.origin}${docsPath}`); + await expect(page.getByRole('heading', { name: '/docs', exact: true })).toBeVisible(); + await expect(visitsValue).toHaveText('1'); + await expect( + page.getByRole('region', { name: 'Links on go.example.org', exact: true }) + ).toHaveCount(0); + + await domainBreadcrumb.click(); + await page.getByRole('link', { name: '/a%2Fb', exact: true }).click(); + await expect(page.getByRole('heading', { name: '/a%2Fb', exact: true })).toBeVisible(); + await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links/a%252Fb`); + await expect(visitsValue).toHaveText('1'); + await domainBreadcrumb.click(); + await page.getByRole('link', { name: '/a/b', exact: true }).click(); + await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links/a/b`); + await expect(page.getByRole('heading', { name: '/a/b', exact: true })).toBeVisible(); + await expect(visitsValue).toHaveText('0'); + + await domainBreadcrumb.click(); + await baseTable.getByRole('link', { name: '/', exact: true }).click(); + await expect(page).toHaveURL(`${appServer.origin}/admin/domains/go.example.org/links`); + await expect(page.getByRole('heading', { name: '/', exact: true })).toBeVisible(); + await expect(visitsValue).toHaveText('0'); + await domainBreadcrumb.click(); + await expect(visitsValue).toHaveText('2'); + + const html = await page.content(); + await expect(html).not.toContain('e2e-only-password'); await page.getByRole('button', { name: 'Sign out', exact: true }).click(); await expect(page).toHaveURL(/\/signin\?/); - await page.goto(`${appServer.origin}/admin`); + await page.goto(`${appServer.origin}${docsPath}`); await expect(page).toHaveURL(/\/signin\?/); await expect(page.getByLabel('Username', { exact: true })).toBeVisible(); }); diff --git a/apps/web/e2e/server.mjs b/apps/web/e2e/server.mjs index 0385b38..51c23a4 100644 --- a/apps/web/e2e/server.mjs +++ b/apps/web/e2e/server.mjs @@ -1,11 +1,12 @@ import { mkdir, writeFile } from 'node:fs/promises'; +import { fork } from 'node:child_process'; +import { insertVisit, openDb, runMigrations } from '@goto/db'; import { createServer } from 'node:net'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; const directory = process.env.GOTO_E2E_BUILD_DIR; if (!directory) throw new Error('Missing E2E build directory'); - const runtimeDirectory = join(directory, `runtime-${process.pid}`); await mkdir(join(runtimeDirectory, 'config'), { recursive: true }); process.chdir(runtimeDirectory); @@ -22,27 +23,86 @@ await new Promise((resolve, reject) => reservation.close((error) => (error ? reject(error) : resolve())) ); process.env.PORT = String(address.port); -await writeFile( - join(runtimeDirectory, 'config/goto.yaml'), - JSON.stringify({ - site: { url: `http://localhost:${address.port}` }, - auth: { credentials: { username: 'example', password: 'e2e-only-password' } }, - domains: { - 'go.example.org': { - links: { - '/schema.json': { url: 'https://example.org/public-schema' }, - '/_app/version.json': { url: 'https://example.org/public-asset' }, - '/a/b': { url: 'https://example.org/decoded' }, - '/a%2Fb': { url: 'https://example.org/literal-percent' }, - '/docs': { url: 'https://example.org/first-domain' }, - '/healthz': { url: 'https://example.org/public-health' } - } - }, - 'tiny.example.org': { - links: { '/docs': { url: 'https://example.org/second-domain' } } +const configuration = { + site: { url: `http://localhost:${address.port}` }, + auth: { credentials: { username: 'example', password: 'e2e-only-password' } }, + domains: { + 'go.example.org': { + links: { + '/schema.json': { url: 'https://example.org/public-schema' }, + '/_app/version.json': { url: 'https://example.org/public-asset' }, + '/a/b': { url: 'https://example.org/decoded' }, + '/a%2Fb': { url: 'https://example.org/literal-percent' }, + '/docs': { url: 'https://example.org/first-domain' }, + '/healthz': { url: 'https://example.org/public-health' } } - } - }) -); - + }, + 'tiny.example.org': { links: { '/docs': { url: 'https://example.org/second-domain' } } } + } +}; +await writeFile(join(runtimeDirectory, 'config/goto.yaml'), JSON.stringify(configuration)); +if (process.env.GOTO_E2E_WORKER === '1') { + await seedVisits(); + await startWorker(); +} await import(pathToFileURL(join(directory, 'server.js')).href); + +async function seedVisits() { + const db = await openDb(join(runtimeDirectory, 'data/goto.sqlite')); + try { + await runMigrations(db); + const common = { + visited_at: Date.now() - 1000, + ip: '203.0.113.1', + user_agent: null, + accept_language: null, + browser: 'Other', + os: 'Other' + }; + const visits = [ + { domain: 'go.example.org', path: '/docs', outcome: 'valid' }, + { domain: 'go.example.org', path: '/a/b', outcome: 'invalid' }, + { domain: 'go.example.org', path: '/', outcome: 'invalid' }, + { domain: 'go.example.org', path: '/a%2Fb', outcome: 'valid' }, + { domain: 'tiny.example.org', path: '/docs', outcome: 'valid' } + ]; + for (const visit of visits) await insertVisit(db, { ...common, ...visit }); + } finally { + await db.destroy(); + } +} + +async function startWorker() { + const worker = fork(new URL('../../worker/dist/index.js', import.meta.url), [], { + cwd: runtimeDirectory, + env: { ...process.env, HOST: '127.0.0.1', PORT: '0', SHUTDOWN_TIMEOUT: '5' }, + execArgv: [], + stdio: ['ignore', 'inherit', 'inherit', 'ipc'] + }); + process.on('SIGTERM', () => worker.kill('SIGTERM')); + process.on('SIGINT', () => worker.kill('SIGTERM')); + process.on('disconnect', () => worker.kill('SIGTERM')); + process.once('exit', () => worker.kill('SIGKILL')); + worker.once('exit', (code, signal) => { + if (code !== 0 || signal !== null) process.exitCode = 1; + }); + const ready = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + worker.kill('SIGKILL'); + reject(new Error('E2E worker startup timed out')); + }, 15_000); + worker.once('message', (message) => { + clearTimeout(timer); + resolve(message); + }); + worker.once('error', (error) => { + clearTimeout(timer); + reject(error); + }); + worker.once('exit', () => { + clearTimeout(timer); + reject(new Error('E2E worker exited before readiness')); + }); + }); + if (ready.type !== 'ready') throw new Error('Invalid worker readiness'); +} diff --git a/apps/web/e2e/support/test.ts b/apps/web/e2e/support/test.ts index a3a1814..7546b55 100644 --- a/apps/web/e2e/support/test.ts +++ b/apps/web/e2e/support/test.ts @@ -53,13 +53,17 @@ async function stopServer(child: ChildProcess): Promise { } } -export const test = base.extend<{ appServer: AppServer }>({ - // eslint-disable-next-line no-empty-pattern -- Playwright discovers dependencies by destructuring. - appServer: async ({}, use, testInfo) => { +export const test = base.extend<{ appServer: AppServer; withWorker: boolean }>({ + withWorker: [false, { option: true }], + appServer: async ({ withWorker }, use, testInfo) => { const directory = process.env.GOTO_E2E_BUILD_DIR; if (!directory) throw new Error('Missing E2E build directory'); const child = fork(new URL('../server.mjs', import.meta.url), [], { - env: { ...serverEnvironment(), GOTO_E2E_BUILD_DIR: directory }, + env: { + ...serverEnvironment(), + GOTO_E2E_BUILD_DIR: directory, + GOTO_E2E_WORKER: withWorker ? '1' : '0' + }, execArgv: [], stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }); diff --git a/apps/web/package.json b/apps/web/package.json index a27dc91..7ef8ae0 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -21,7 +21,7 @@ "format": "prettier --ignore-path ../../.prettierignore --write .", "test": "vitest run", "test:coverage": "vitest run --coverage", - "test:e2e": "playwright test" + "test:e2e": "pnpm --filter @goto/worker... build && playwright test" }, "dependencies": { "@goto/config": "workspace:*", diff --git a/apps/web/src/lib/components/AdminNav.svelte b/apps/web/src/lib/components/admin/AdminNav.svelte similarity index 94% rename from apps/web/src/lib/components/AdminNav.svelte rename to apps/web/src/lib/components/admin/AdminNav.svelte index 4c98480..a27a8c3 100644 --- a/apps/web/src/lib/components/AdminNav.svelte +++ b/apps/web/src/lib/components/admin/AdminNav.svelte @@ -2,7 +2,7 @@ import { page } from '$app/state'; import { goto } from '$app/navigation'; import { authClient } from '#lib/auth-client.js'; - import ThemeToggle from './ThemeToggle.svelte'; + import ThemeToggle from '../ThemeToggle.svelte'; let { title }: { title: string } = $props(); @@ -49,7 +49,8 @@
diff --git a/apps/web/src/lib/components/admin/AdminPage.svelte b/apps/web/src/lib/components/admin/AdminPage.svelte new file mode 100644 index 0000000..184b496 --- /dev/null +++ b/apps/web/src/lib/components/admin/AdminPage.svelte @@ -0,0 +1,39 @@ + + +{title} — {siteTitle} + +
+
+ {#if backHref}{/if} +
+

{title}

+ {#if updatedAt !== null} + + {/if} +
+
+ {@render children()} +
diff --git a/apps/web/src/lib/components/admin/ReportNotice.svelte b/apps/web/src/lib/components/admin/ReportNotice.svelte new file mode 100644 index 0000000..1011e06 --- /dev/null +++ b/apps/web/src/lib/components/admin/ReportNotice.svelte @@ -0,0 +1,10 @@ + + +{#if !collectionEnabled} +

Analytics disabled.

+{:else if !available} +

Statistics unavailable.

+{/if} diff --git a/apps/web/src/lib/components/admin/StatCard.svelte b/apps/web/src/lib/components/admin/StatCard.svelte new file mode 100644 index 0000000..aa39dce --- /dev/null +++ b/apps/web/src/lib/components/admin/StatCard.svelte @@ -0,0 +1,9 @@ + + +
+ {typeof value === 'string' ? value : formatCount(value)} + {label} +
diff --git a/apps/web/src/lib/components/admin/format.test.ts b/apps/web/src/lib/components/admin/format.test.ts new file mode 100644 index 0000000..dbf6981 --- /dev/null +++ b/apps/web/src/lib/components/admin/format.test.ts @@ -0,0 +1,13 @@ +import { expect, test } from 'vitest'; +import { formatUpdated, formatRetention } from './format.js'; + +test('formats the snapshot update time in UTC', () => { + const updatedAt = Date.parse('2026-11-06T22:03:00Z'); + expect(formatUpdated(updatedAt)).toBe('Updated Nov 6, 2026, 10:03 PM UTC'); + expect(formatUpdated(null)).toBe(''); +}); + +test('shows the configured retention policy rather than an elapsed day count', () => { + expect(formatRetention(3)).toBe('3 months'); + expect(formatRetention(0)).toBe('Off'); +}); diff --git a/apps/web/src/lib/components/admin/format.ts b/apps/web/src/lib/components/admin/format.ts new file mode 100644 index 0000000..6d8cd5e --- /dev/null +++ b/apps/web/src/lib/components/admin/format.ts @@ -0,0 +1,17 @@ +export function formatCount(value: number | null): string { + return value === null ? '—' : value.toLocaleString('en'); +} + +export function formatUpdated(value: number | null): string { + if (value === null) return ''; + const formatter = new Intl.DateTimeFormat('en', { + dateStyle: 'medium', + timeStyle: 'short', + timeZone: 'UTC' + }); + return `Updated ${formatter.format(value)} UTC`; +} + +export function formatRetention(months: number): string { + return months === 0 ? 'Off' : `${months} months`; +} diff --git a/apps/web/src/lib/server/link-state.ts b/apps/web/src/lib/server/link-state.ts new file mode 100644 index 0000000..7c82464 --- /dev/null +++ b/apps/web/src/lib/server/link-state.ts @@ -0,0 +1,9 @@ +import type { PreparedLink } from '@goto/config'; +import type { Immutable } from './snapshot.js'; + +export function currentLinkState(link: Immutable, now: number): string { + if (!link.enabled) return 'Disabled'; + if (link.valid_from !== undefined && now < Date.parse(link.valid_from)) return 'Scheduled'; + if (link.valid_until !== undefined && now >= Date.parse(link.valid_until)) return 'Expired'; + return 'Active'; +} diff --git a/apps/web/src/lib/server/visit-reports.test.ts b/apps/web/src/lib/server/visit-reports.test.ts new file mode 100644 index 0000000..2ad5d88 --- /dev/null +++ b/apps/web/src/lib/server/visit-reports.test.ts @@ -0,0 +1,91 @@ +import { access, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { GotoConfiguration } from '@goto/config'; +import { openDb, runMigrations } from '@goto/db'; +import { publishVisitReports, type PreparedVisitReport } from '@goto/db/visit-reports'; +import { expect, test, vi } from 'vitest'; +import { readVisitReport } from './visit-reports.js'; + +const report: PreparedVisitReport = { + totals: { total: 3, valid: 2, invalid: 1, base: 1 }, + daily: [], + weekly: [], + monthly: [], + browsers: [], + operatingSystems: [], + domainCounts: [], + linkCounts: [] +}; + +test('reads prepared output and hides snapshots from a different retention policy', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goto-page-reports-')); + const configuration: Pick = { + database: { + path: join(directory, 'goto.sqlite'), + worker_path: join(directory, 'worker.sqlite') + }, + analytics: { retention_months: 3 } + }; + try { + const db = await openDb(configuration.database.path); + try { + await runMigrations(db); + const period = { + timezone: 'UTC' as const, + retentionMonths: 3 as const, + since: 100, + asOf: 200, + days: [], + weeks: [], + months: [] + }; + await publishVisitReports(db, period, [ + { key: 'overview', data: JSON.stringify(report) }, + { key: 'empty', data: JSON.stringify(report) } + ]); + } finally { + await db.destroy(); + } + expect((await readVisitReport(configuration))?.report.totals.total).toBe(3); + configuration.analytics.retention_months = 9; + expect(await readVisitReport(configuration)).toBeNull(); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('enabled collection fails safely without creating missing storage', async () => { + const directory = await mkdtemp(join(tmpdir(), 'goto-missing-reports-')); + const missing = join(directory, 'missing'); + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const configuration = { + database: { + path: join(missing, 'goto.sqlite'), + worker_path: join(directory, 'worker.sqlite') + }, + analytics: { retention_months: 3 as const } + }; + expect(await readVisitReport(configuration)).toBeNull(); + expect(warning).toHaveBeenCalledExactlyOnceWith('Prepared visit reports unavailable'); + await expect(access(missing)).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + warning.mockRestore(); + await rm(directory, { recursive: true, force: true }); + } +}); + +test('disabled collection does not attempt to open missing storage', async () => { + const warning = vi.spyOn(console, 'warn').mockImplementation(() => {}); + try { + const configuration = { + database: { path: '/missing/not-used.sqlite', worker_path: '/missing/worker.sqlite' }, + analytics: { retention_months: 0 as const } + }; + expect(await readVisitReport(configuration)).toBeNull(); + expect(warning).not.toHaveBeenCalled(); + } finally { + warning.mockRestore(); + } +}); diff --git a/apps/web/src/lib/server/visit-reports.ts b/apps/web/src/lib/server/visit-reports.ts new file mode 100644 index 0000000..60f4fc9 --- /dev/null +++ b/apps/web/src/lib/server/visit-reports.ts @@ -0,0 +1,35 @@ +import type { GotoConfiguration } from '@goto/config'; +import { openDb } from '@goto/db'; +import { readPreparedVisitReports, type StoredVisitReports } from '@goto/db/visit-reports'; +import type { Immutable } from './snapshot.js'; + +type ReportConfiguration = Immutable>; + +export async function readVisitReport( + configuration: ReportConfiguration, + domain?: string, + path?: string +): Promise { + const months = configuration.analytics.retention_months; + if (months === 0) return null; + let db: Awaited> | undefined; + try { + db = await openDb(configuration.database.path, { readOnly: true }); + const reports = await readPreparedVisitReports(db, domain, path); + if (!reports || reports.period.retentionMonths !== months) return null; + return reports; + } catch { + warnUnavailable(); + return null; + } finally { + await db?.destroy().catch(warnUnavailable); + } +} + +function warnUnavailable(): void { + try { + console.warn('Prepared visit reports unavailable'); + } catch { + return; + } +} diff --git a/apps/web/src/lib/styles/admin.css b/apps/web/src/lib/styles/admin.css new file mode 100644 index 0000000..7e7948b --- /dev/null +++ b/apps/web/src/lib/styles/admin.css @@ -0,0 +1,155 @@ +.admin-page { + max-width: var(--content-admin-width); + margin-inline: auto; + padding: var(--space-8) var(--space-5); + display: flex; + flex-direction: column; + gap: var(--space-8); +} +.admin-heading { + display: flex; + flex-direction: column; + gap: var(--space-4); +} +.admin-title-row { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: var(--space-4) var(--space-5); +} +.admin-updated { + margin-inline-start: auto; + text-align: end; + color: var(--color-text-muted); +} +.admin-heading h1 { + font-size: var(--font-size-heading); + line-height: var(--line-height-heading); + overflow-wrap: anywhere; +} +.admin-page h2 { + font-size: 1.25rem; + margin-bottom: var(--space-4); + overflow-wrap: anywhere; +} +.admin-page a { + color: var(--color-primary); +} +.admin-cards { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr)); + gap: var(--space-5); +} +.admin-cards-summary { + grid-template-columns: repeat(4, minmax(0, 1fr)); +} +.admin-stat, +.admin-info { + background: var(--color-surface-card); + border: var(--border-width) solid var(--color-border-subtle); + border-radius: var(--radius-card); + padding: var(--space-5); +} +.admin-stat { + display: flex; + flex-direction: column; + gap: var(--space-4); +} +.admin-stat-value { + font-size: var(--font-size-heading); + font-weight: 700; +} +.admin-stat-label { + font-weight: 600; +} +.admin-notice { + padding: var(--space-5); + border: var(--border-width) solid var(--color-border-strong); + border-radius: var(--radius-card); +} +.admin-table-scroll { + overflow-x: auto; + border: var(--border-width) solid var(--color-border-subtle); + border-radius: var(--radius-card); + background: var(--color-surface-card); +} +.admin-table { + width: 100%; + border-collapse: collapse; + font-size: var(--font-size-sm); +} +.admin-table th, +.admin-table td { + padding: var(--space-4); + text-align: start; + vertical-align: top; + border-top: var(--border-width) solid var(--color-border-subtle); + overflow-wrap: anywhere; +} +.admin-table thead { + background: var(--color-surface-muted); +} +.admin-link-table { + table-layout: fixed; + min-width: 42rem; +} +.admin-link-table thead th:nth-child(1) { + width: 18%; +} +.admin-link-table thead th:nth-child(2) { + width: 46%; +} +.admin-link-table thead th:nth-child(3) { + width: 20%; +} +.admin-link-table thead th:nth-child(4) { + width: 16%; +} +.admin-table tbody tr { + position: relative; +} +.admin-table tbody tr:hover, +.admin-table tbody tr:focus-within { + background: var(--color-surface-muted); +} +.admin-row-link::after { + content: ''; + position: absolute; + inset: 0; +} +.admin-row-link:focus-visible { + outline: none; +} +.admin-row-link:focus-visible::after { + outline: var(--outline-focus); + outline-offset: -2px; +} +.admin-destination { + position: relative; + z-index: 1; +} +.admin-info dl { + display: grid; + grid-template-columns: minmax(7rem, auto) minmax(0, 1fr); + gap: var(--space-4) var(--space-5); +} +.admin-info dt { + color: var(--color-text-muted); +} +.admin-info dd { + margin: 0; + overflow-wrap: anywhere; +} +@media (max-width: 640px) { + .admin-cards-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .admin-info dl { + grid-template-columns: 1fr; + } +} +@media (max-width: 400px) { + .admin-cards-summary { + grid-template-columns: 1fr; + } +} diff --git a/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts index 8f77130..2ae8e84 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts +++ b/apps/web/src/routes/(admin)/(auth)/admin/+page.server.ts @@ -1,14 +1,31 @@ +import { readVisitReport } from '#lib/server/visit-reports.js'; +import type { StoredVisitReports } from '@goto/db/visit-reports'; import type { PageServerLoad } from './$types'; -export const load: PageServerLoad = ({ locals }) => ({ - domains: Object.entries(locals.snapshot.domains).map(([domain, prepared]) => ({ +export const load: PageServerLoad = async ({ locals }) => { + const snapshot = locals.snapshot; + const configuration = snapshot.configuration; + const retentionMonths = configuration.analytics.retention_months; + const collectionEnabled = retentionMonths !== 0; + let preparedReports: StoredVisitReports | null = null; + if (collectionEnabled) preparedReports = await readVisitReport(configuration); + const report = preparedReports?.report; + + const domains = Object.entries(snapshot.domains).map(([domain, prepared]) => ({ domain, - links: Object.entries(prepared.links).map(([path, link]) => ({ - path, - title: link.title, - url: link.url, - enabled: link.enabled, - status_code: link.status_code - })) - })) -}); + linkCount: Object.keys(prepared.links).length, + counts: report?.domainCounts.find((row) => row.domain === domain)?.counts ?? null, + href: `/admin/domains/${encodeURIComponent(domain)}` + })); + return { + collectionEnabled, + available: preparedReports !== null, + retentionMonths, + since: preparedReports?.period.since ?? null, + asOf: preparedReports?.period.asOf ?? null, + domainCount: domains.length, + linkCount: domains.reduce((sum, row) => sum + row.linkCount, 0), + totals: report?.totals ?? null, + domains + }; +}; diff --git a/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte index c319cdf..34de92e 100644 --- a/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte +++ b/apps/web/src/routes/(admin)/(auth)/admin/+page.svelte @@ -1,127 +1,42 @@ - - Configured links — {data.title} - - - - -
-
-

Configured links

-

Read-only. Edit the YAML configuration to change links.

-
- - {#if data.domains.length === 0} -

No links configured.

- {:else} - {#each data.domains as group (group.domain)} -
-

{group.domain}

-
- - - + + +
+ + + + +
+
+

Domains

+ {#if data.domains.length === 0}

No domains configured.

+ {:else} +
+
Configured links on {group.domain}
+ + + {#each data.domains as row (row.domain)} - - - - - + + + - - - {#each group.links as link (link.path)} - - - - - - - - {/each} - -
DomainLinksVisits
PathTitleDestinationEnabledHTTP status{row.domain}{formatCount(row.linkCount)}{formatCount(row.counts?.valid ?? null)}
{link.path}{link.title}{link.url}{link.enabled ? 'Yes' : 'No'}{link.status_code}
-
-
- {/each} - {/if} -
- - + {/each} + + +
+ {/if} + + diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts new file mode 100644 index 0000000..c237cda --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.server.ts @@ -0,0 +1,64 @@ +import { error } from '@sveltejs/kit'; +import type { VisitCounts, StoredVisitReports } from '@goto/db/visit-reports'; +import { currentLinkState } from '#lib/server/link-state.js'; +import { readVisitReport } from '#lib/server/visit-reports.js'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = async ({ locals, params }) => { + const snapshot = locals.snapshot; + const domain = params.domain; + if (!Object.hasOwn(snapshot.domains, domain)) throw error(404, 'Domain not configured'); + const prepared = snapshot.domains[domain]!; + const configuration = snapshot.configuration; + const collectionEnabled = configuration.analytics.retention_months !== 0; + let preparedReports: StoredVisitReports | null = null; + if (collectionEnabled) preparedReports = await readVisitReport(configuration, domain); + const report = preparedReports?.report; + + const countsByPath = new Map(); + for (const row of report?.linkCounts ?? []) countsByPath.set(row.path, row.counts); + const baseCounts = countsByPath.get('/') ?? null; + let invalidClicks: number | null = null; + if (report && baseCounts) invalidClicks = report.totals.invalid - baseCounts.invalid; + const now = Date.now(); + const links = Object.entries(prepared.links).map(([path, link]) => ({ + path, + url: link.url, + state: currentLinkState(link, now), + counts: countsByPath.get(path) ?? null, + href: linkHref(domain, path) + })); + const root = prepared.links['/']; + let baseUrl = prepared.fallback === 404 ? null : prepared.fallback; + let baseState = 'Fallback'; + if (root && currentLinkState(root, now) === 'Active') { + baseUrl = root.url; + baseState = 'Active'; + } + + return { + domain, + retentionMonths: configuration.analytics.retention_months, + collectionEnabled, + available: preparedReports !== null, + since: preparedReports?.period.since ?? null, + asOf: preparedReports?.period.asOf ?? null, + linkCount: links.length, + totals: report?.totals ?? null, + invalidClicks, + base: { + url: baseUrl, + state: baseState, + counts: baseCounts, + href: linkHref(domain, '/') + }, + links + }; +}; + +function linkHref(domain: string, path: string): string { + const base = `/admin/domains/${encodeURIComponent(domain)}/links`; + if (path === '/') return base; + const segments = path.slice(1).split('/').map(encodeURIComponent); + return `${base}/${segments.join('/')}`; +} diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte new file mode 100644 index 0000000..b622bec --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/+page.svelte @@ -0,0 +1,71 @@ + + + + +
+ + + + +
+
+

Base URL

+
+ + + + + + + + + + +
PathDestinationStateVisits
/{#if data.base.url}{data.base.url}{:else}404 Not found{/if}{data.base.state}{formatCount(data.base.counts?.valid ?? null)}
+
+
+
+

Links

+ {#if data.links.length === 0}

No links configured.

+ {:else} +
+ + + + {#each data.links as link (link.path)} + + + + + + {/each} + +
PathDestinationStateVisits
{link.path}{link.url}{link.state}{formatCount(link.counts?.valid ?? null)}
+
+ {/if} +
+ diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts new file mode 100644 index 0000000..c5baf33 --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.server.ts @@ -0,0 +1,36 @@ +import { error } from '@sveltejs/kit'; +import type { StoredVisitReports } from '@goto/db/visit-reports'; +import { currentLinkState } from '#lib/server/link-state.js'; +import { readVisitReport } from '#lib/server/visit-reports.js'; +import type { PageServerLoad } from './$types'; + +export const load: PageServerLoad = async ({ locals, params }) => { + const snapshot = locals.snapshot; + const domain = params.domain; + if (!Object.hasOwn(snapshot.domains, domain)) throw error(404, 'Domain not configured'); + const prepared = snapshot.domains[domain]!; + // Kit has decoded route parameters; preserve literal percent paths as-is. + const path = `/${params.path ?? ''}`; + if (path !== '/' && !Object.hasOwn(prepared.links, path)) throw error(404, 'Link not configured'); + const link = prepared.links[path]; + const configuration = snapshot.configuration; + const collectionEnabled = configuration.analytics.retention_months !== 0; + let reports: StoredVisitReports | null = null; + if (collectionEnabled) reports = await readVisitReport(configuration, domain, path); + const state = link ? currentLinkState(link, Date.now()) : 'Fallback'; + + return { + domain, + path, + domainHref: `/admin/domains/${encodeURIComponent(domain)}`, + url: link?.url ?? null, + fallback: prepared.fallback, + state, + retentionMonths: configuration.analytics.retention_months, + collectionEnabled, + available: reports !== null, + since: reports?.period.since ?? null, + asOf: reports?.period.asOf ?? null, + totals: reports?.selection.totals ?? null + }; +}; diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte new file mode 100644 index 0000000..e1b52dd --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/+page.svelte @@ -0,0 +1,33 @@ + + + + +
+ + +
+
+
+
Destination
+
+ {#if data.url}{data.url} + {:else if data.fallback === 404}404 Not found{:else}{data.fallback}{/if} +
+
State
+
{data.state}
+
+
+ diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts new file mode 100644 index 0000000..10d1679 --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/links/[...path]/page.server.test.ts @@ -0,0 +1,105 @@ +import { validateConfig } from '@goto/config'; +import type { PreparedVisitReport } from '@goto/db/visit-reports'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { prepareSnapshot } from '#lib/server/snapshot.js'; +import { readVisitReport } from '#lib/server/visit-reports.js'; +import { load } from './+page.server.js'; + +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +beforeEach(() => vi.resetAllMocks()); + +const domain = 'go.example.test'; +function request(path = '/docs', host = domain, retention_months = 3) { + const prepared = validateConfig( + { + auth: { credentials: { username: 'test', password: 'link-test-password' } }, + analytics: { retention_months }, + domains: { + [domain]: { + links: { + '/docs': { url: 'https://example.test/docs' }, + '/a/b': { url: 'https://example.test/nested' }, + '/a%2Fb': { url: 'https://example.test/percent', enabled: false } + } + } + } + }, + { env: {} } + ); + return { + locals: { snapshot: prepareSnapshot(prepared) }, + params: { domain: host, path: path.slice(1) } + } as Parameters[0]; +} +const report: PreparedVisitReport = { + totals: { total: 6, valid: 3, invalid: 3, base: 1 }, + daily: [], + weekly: [], + monthly: [], + browsers: [], + operatingSystems: [], + domainCounts: [], + linkCounts: [] +}; +const period = { + timezone: 'UTC' as const, + retentionMonths: 3 as const, + since: 100, + asOf: 200, + days: [], + weeks: [], + months: [] +}; + +test('loads the requested path report and current destination with a domain breadcrumb', async () => { + const totals = { total: 3, valid: 2, invalid: 1, base: 0 }; + vi.mocked(readVisitReport).mockResolvedValue({ + period, + report, + selection: { ...report, totals } + }); + const data = await load(request()); + if (!data) throw new Error('Missing link page'); + expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/docs'); + expect(data.path).toBe('/docs'); + expect(data.totals).toEqual(totals); + expect(data.url).toBe('https://example.test/docs'); + expect(data.state).toBe('Active'); + expect(data.domainHref).toBe('/admin/domains/go.example.test'); + expect(data.asOf).toBe(200); + expect(data.retentionMonths).toBe(3); +}); + +test('keeps decoded nested and literal-percent paths distinct and supports the unconfigured root', async () => { + vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report }); + const nested = await load(request('/a/b')); + const percent = await load(request('/a%2Fb')); + const root = await load(request('/')); + if (!nested || !percent || !root) throw new Error('Missing path page'); + expect(nested.path).toBe('/a/b'); + expect(percent.path).toBe('/a%2Fb'); + expect(percent.state).toBe('Disabled'); + expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain, '/a%2Fb'); + expect(root.path).toBe('/'); + expect(root.state).toBe('Fallback'); + expect(root.url).toBeNull(); + expect(root.fallback).toBe(404); +}); + +test('rejects unknown scopes before reading and keeps unavailable or disabled counts empty', async () => { + await expect(load(request('/docs', 'unknown.test'))).rejects.toMatchObject({ status: 404 }); + await expect(load(request('/not-configured'))).rejects.toMatchObject({ status: 404 }); + expect(readVisitReport).not.toHaveBeenCalled(); + vi.mocked(readVisitReport).mockResolvedValue(null); + const missing = await load(request()); + if (!missing) throw new Error('Missing unavailable page'); + expect(missing.available).toBe(false); + expect(missing.totals).toBeNull(); + vi.clearAllMocks(); + const disabled = await load(request('/docs', domain, 0)); + if (!disabled) throw new Error('Missing disabled page'); + expect(disabled.collectionEnabled).toBe(false); + expect(disabled.retentionMonths).toBe(0); + expect(disabled.totals).toBeNull(); + expect(readVisitReport).not.toHaveBeenCalled(); +}); diff --git a/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts new file mode 100644 index 0000000..fc3ea2d --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/domains/[domain]/page.server.test.ts @@ -0,0 +1,121 @@ +import { validateConfig } from '@goto/config'; +import type { PreparedVisitReport, StoredVisitReports } from '@goto/db/visit-reports'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { prepareSnapshot } from '#lib/server/snapshot.js'; +import { readVisitReport } from '#lib/server/visit-reports.js'; +import { load } from './+page.server.js'; + +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +beforeEach(() => vi.resetAllMocks()); + +const domain = 'go.example.test'; +function request(host = domain, rootUrl?: string) { + const links: Record = { + '/docs': { url: 'https://example.test/docs' }, + '/a/b': { url: 'https://example.test/nested' }, + '/a%2Fb': { url: 'https://example.test/percent', enabled: false } + }; + if (rootUrl) links['/'] = { url: rootUrl }; + const prepared = validateConfig( + { + auth: { credentials: { username: 'test', password: 'domain-test-password' } }, + domains: { [domain]: { links } } + }, + { env: {} } + ); + return { + locals: { snapshot: prepareSnapshot(prepared) }, + params: { domain: host } + } as Parameters[0]; +} +const docs = { total: 3, valid: 2, invalid: 1, base: 0 }; +const root = { total: 1, valid: 0, invalid: 1, base: 1 }; +const empty = { total: 0, valid: 0, invalid: 0, base: 0 }; +const report: PreparedVisitReport = { + totals: { total: 6, valid: 3, invalid: 3, base: 1 }, + daily: [], + weekly: [], + monthly: [], + browsers: [], + operatingSystems: [], + domainCounts: [], + linkCounts: [ + { path: '/docs', counts: docs }, + { path: '/', counts: root }, + { path: '/a%2Fb', counts: empty } + ] +}; +const period = { + timezone: 'UTC' as const, + retentionMonths: 3 as const, + since: 100, + asOf: 200, + days: [], + weeks: [], + months: [] +}; + +test('reads only domain-wide traffic and links to dedicated path routes', async () => { + const reports: StoredVisitReports = { period, report, selection: report }; + vi.mocked(readVisitReport).mockResolvedValue(reports); + const data = await load(request()); + if (!data) throw new Error('Missing page data'); + expect(readVisitReport).toHaveBeenCalledWith(expect.any(Object), domain); + expect(data.totals).toEqual(report.totals); + expect(data.invalidClicks).toBe(2); + expect(data.linkCount).toBe(3); + expect(data.retentionMonths).toBe(3); + expect(data.links).toMatchObject([ + { href: '/admin/domains/go.example.test/links/docs' }, + { href: '/admin/domains/go.example.test/links/a/b' }, + { href: '/admin/domains/go.example.test/links/a%252Fb' } + ]); + expect(data.links[2]?.counts).toEqual(empty); + expect(data.links[2]?.state).toBe('Disabled'); + expect(data.base.href).toBe('/admin/domains/go.example.test/links'); + expect(data.base.url).toBeNull(); + expect(data.base.state).toBe('Fallback'); + expect(data.base.counts).toEqual(root); + expect(data.asOf).toBe(200); +}); + +test('shows an active base destination without reclassifying historical outcomes', async () => { + vi.mocked(readVisitReport).mockResolvedValue({ period, report, selection: report }); + const data = await load(request(domain, 'https://example.test/start')); + if (!data) throw new Error('Missing base page data'); + expect(data.base.url).toBe('https://example.test/start'); + expect(data.base.state).toBe('Active'); + expect(data.base.counts).toEqual(root); +}); + +test('excludes invalid base traffic without subtracting valid base redirects', async () => { + const mixedRoot = { total: 5, valid: 3, invalid: 2, base: 5 }; + const mixedReport: PreparedVisitReport = { + ...report, + totals: { total: 10, valid: 6, invalid: 4, base: 5 }, + linkCounts: report.linkCounts.map((row) => + row.path === '/' ? { ...row, counts: mixedRoot } : row + ) + }; + vi.mocked(readVisitReport).mockResolvedValue({ + period, + report: mixedReport, + selection: mixedReport + }); + const data = await load(request()); + if (!data) throw new Error('Missing domain page data'); + expect(data.invalidClicks).toBe(2); + expect(data.totals?.valid).toBe(6); +}); + +test('rejects unknown domains before reading and preserves unavailable rather than zero', async () => { + await expect(load(request('unknown.test'))).rejects.toMatchObject({ status: 404 }); + expect(readVisitReport).not.toHaveBeenCalled(); + vi.mocked(readVisitReport).mockResolvedValue(null); + const data = await load(request()); + if (!data) throw new Error('Missing unavailable page'); + expect(data.available).toBe(false); + expect(data.totals).toBeNull(); + expect(data.invalidClicks).toBeNull(); + expect(data.links[0]?.counts).toBeNull(); +}); diff --git a/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts b/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts new file mode 100644 index 0000000..3991596 --- /dev/null +++ b/apps/web/src/routes/(admin)/(auth)/admin/page.server.test.ts @@ -0,0 +1,76 @@ +import { validateConfig } from '@goto/config'; +import type { StoredVisitReports } from '@goto/db/visit-reports'; +import { beforeEach, expect, test, vi } from 'vitest'; +import { prepareSnapshot } from '#lib/server/snapshot.js'; +import { readVisitReport } from '#lib/server/visit-reports.js'; +import { load } from './+page.server.js'; + +vi.mock('#lib/server/visit-reports.js', () => ({ readVisitReport: vi.fn() })); +beforeEach(() => vi.resetAllMocks()); + +function request(retention_months = 3) { + const prepared = validateConfig( + { + auth: { credentials: { username: 'test', password: 'overview-test-password' } }, + analytics: { retention_months }, + domains: { + 'go.example.test': { + links: { + '/docs': { url: 'https://example.test/docs' }, + '/disabled': { url: 'https://example.test/disabled', enabled: false } + } + } + } + }, + { env: {} } + ); + return { locals: { snapshot: prepareSnapshot(prepared) } } as Parameters[0]; +} + +test('shows current configuration and the worker-prepared snapshot without re-aggregating', async () => { + const counts = { total: 4, valid: 2, invalid: 2, base: 1 }; + const report = { + totals: { ...counts, total: 9, valid: 7 }, + daily: [], + weekly: [], + monthly: [], + browsers: [], + operatingSystems: [], + domainCounts: [{ domain: 'go.example.test', counts }], + linkCounts: [] + }; + const prepared: StoredVisitReports = { + period: { + timezone: 'UTC', + retentionMonths: 3, + since: 100, + asOf: 200, + days: [], + weeks: [], + months: [] + }, + report, + selection: report + }; + vi.mocked(readVisitReport).mockResolvedValue(prepared); + const data = await load(request()); + if (!data) throw new Error('Missing page data'); + expect(data.domainCount).toBe(1); + expect(data.linkCount).toBe(2); + expect(data.totals?.total).toBe(9); + expect(data.asOf).toBe(200); + expect(data.domains[0]?.href).toBe('/admin/domains/go.example.test'); +}); + +test('disabled collection does not read a report; missing preparation is not zero', async () => { + const disabled = await load(request(0)); + if (!disabled) throw new Error('Missing disabled page'); + expect(disabled.collectionEnabled).toBe(false); + expect(disabled.totals).toBeNull(); + expect(readVisitReport).not.toHaveBeenCalled(); + vi.mocked(readVisitReport).mockResolvedValue(null); + const unavailable = await load(request()); + if (!unavailable) throw new Error('Missing unavailable page'); + expect(unavailable.available).toBe(false); + expect(unavailable.domains[0]?.counts).toBeNull(); +}); -- 2.54.0