Make attachment highlighting opt-in #199

Merged
eleith merged 1 commits from web/opt-in-highlighting into main 2026-11-07 16:06:54 +00:00
10 changed files with 411 additions and 223 deletions

No files matched your search

@@ -0,0 +1,6 @@
---
"@luzzle/web.worker": patch
"@luzzle/web": patch
---
Require fields.highlight selectors for attachment highlighting. Skip oversized, NUL-containing or invalid UTF-8 sources and oversized generated HTML, leaving original attachments downloadable.
+2
View File
@@ -157,6 +157,8 @@ pieces:
tags: keywords
attachments:
- files[*].file
highlight:
- files[*].file
- type: words
templates:
icon:
@@ -35,6 +35,22 @@ test("consumers can supply complete extended schemas with their own defaults", (
expect(config.pieces[0].badge).toBe("new");
});
test("attachment highlighting requires explicit selectors", () => {
const fields = { ...piece.fields, attachments: ["files[*].file"] };
const config = validateConfig({ pieces: [{ ...piece, fields }] }, {});
expect(config.pieces[0].fields.highlight).toBeUndefined();
const selected = validateConfig(
{
pieces: [
{ ...piece, fields: { ...fields, highlight: ["files[*].file"] } },
],
},
{},
);
expect(selected.pieces[0].fields.highlight).toEqual(["files[*].file"]);
});
test("loading defaults preserves the supplied schema and base config contract", () => {
const config = loadConfig(undefined, { schema });
expect(config.instance).toBe("demo");
@@ -487,6 +487,13 @@ const PiecesSchema = Type.Array(
examples: [["files[*].file"]],
}),
),
highlight: Type.Optional(
Type.Array(Type.String(), {
description:
"Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
examples: [["files[*].file"]],
}),
),
},
{ additionalProperties: false },
),
@@ -598,6 +598,18 @@
"items": {
"type": "string"
}
},
"highlight": {
"description": "Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
"examples": [
[
"files[*].file"
]
],
"type": "array",
"items": {
"type": "string"
}
}
}
}
+12
View File
@@ -598,6 +598,18 @@
"items": {
"type": "string"
}
},
"highlight": {
"description": "Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
"examples": [
[
"files[*].file"
]
],
"type": "array",
"items": {
"type": "string"
}
}
}
},
@@ -1,199 +1,242 @@
import { describe, test, expect, vi, afterEach } from 'vitest'
import { run } from './index.js'
import { codeToHtml } from 'shiki'
import { getLang } from './lang.js'
import type { Config } from '@luzzle/web.config'
import type { WebPieces } from '../../services/db.js'
import { afterEach, beforeEach, expect, test, vi } from 'vitest'
import { createReadStream } from 'node:fs'
import type * as FileSystem from 'node:fs'
import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
import type * as FileSystemPromises from 'node:fs/promises'
import { tmpdir } from 'node:os'
import path from 'node:path'
import { validateConfig } from '@luzzle/web.config'
import type { Pieces } from '@luzzle/core'
import { codeToHtml } from 'shiki'
import type * as Shiki from 'shiki'
import type { TransformInput } from '../utils/types.js'
import { makeLogger } from '../../../test/logger.js'
import { run } from './index.js'
vi.mock('shiki', () => ({
vi.mock('shiki', async (importOriginal) => ({
...(await importOriginal<typeof Shiki>()),
codeToHtml: vi.fn(),
}))
vi.mock('./lang.js', () => ({
getLang: vi.fn(),
}))
const emptyMap = new Map<string, string>()
const makeWebPiece = (json_metadata = '{}'): WebPieces => ({
id: '1',
type: 'books',
date_updated: 100,
date_added: 50,
json_metadata,
file_path: 'book.md',
key: 'key123',
slug: 'my-book',
title: 'My Book',
vi.mock('node:fs', async (importOriginal) => {
const actual = await importOriginal<typeof FileSystem>()
return { ...actual, createReadStream: vi.fn(actual.createReadStream) }
})
vi.mock('node:fs/promises', async (importOriginal) => {
const actual = await importOriginal<typeof FileSystemPromises>()
return { ...actual, stat: vi.fn(actual.stat) }
})
const makeConfig = (attachments?: string[]): Config =>
({
url: { app: 'http://localhost' },
theme: { markdown: { code: { light: 'github-light', dark: 'github-dark' } } },
pieces: [
const source = '.assets/example/main.js'
const output = 'snippets/piece-key/main.js'
const code = 'console.log("hi")'
const html = '<pre class="shiki">highlighted code</pre>'
let directory: string
function makeInput(): TransformInput {
return {
webPiece: {
id: 'example',
key: 'piece-key',
type: 'snippets',
slug: 'example',
title: 'Example',
file_path: 'example.snippets.md',
date_added: 1,
json_metadata: JSON.stringify({ files: [{ file: 'code' }] }),
},
config: validateConfig(
{
type: 'books',
fields: {
title: 'title',
date_consumed: 'date_consumed',
...(attachments ? { attachments } : {}),
},
pieces: [
{
type: 'snippets',
fields: {
title: 'title',
date_consumed: 'date',
attachments: ['files[*].file', 'downloads'],
highlight: ['files[*].file'],
},
},
],
},
{}
),
outDir: directory,
pieces: {} as Pieces,
assetKeyToPath: new Map([['code', source]]),
priorAssets: [
{
asset_key: 'code',
transformation: 'attachment',
asset_path: output,
mime_type: 'text/javascript',
},
],
}) as unknown as Config
logger: makeLogger(),
}
}
const makePieces = (content: string = 'console.log("hi")') =>
({ getPieceAsset: vi.fn().mockResolvedValue(Buffer.from(content)) }) as unknown as Pieces
beforeEach(async () => {
directory = await mkdtemp(path.join(tmpdir(), 'luzzle-highlight-'))
await mkdir(path.dirname(path.join(directory, output)), { recursive: true })
await writeFile(path.join(directory, output), code)
vi.mocked(codeToHtml).mockResolvedValue(html)
})
afterEach(() => {
afterEach(async () => {
vi.clearAllMocks()
await rm(directory, { recursive: true, force: true })
})
describe('transforms/highlight', () => {
test('returns empty array when piece type has no attachments config', async () => {
const pieces = makePieces()
test('does not read or highlight attachments without explicit selection', async () => {
const input = makeInput()
delete input.config.pieces[0].fields.highlight
const records = await run({
webPiece: makeWebPiece('{}'),
config: makeConfig(),
outDir: '/out',
pieces,
assetKeyToPath: emptyMap,
logger: makeLogger(),
})
expect(await run(input)).toEqual([])
expect(createReadStream).not.toHaveBeenCalled()
expect(codeToHtml).not.toHaveBeenCalled()
})
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
test('highlights selected originals once, leaving other attachments unread', async () => {
const input = makeInput()
input.config.pieces[0].fields.highlight?.push('source')
input.webPiece.json_metadata = JSON.stringify({
files: [{ file: 'code' }],
source: 'code',
downloads: ['recording'],
})
input.assetKeyToPath.set('recording', '.assets/example/recording.gif')
input.priorAssets?.push({
asset_key: 'recording',
transformation: 'attachment',
asset_path: 'snippets/piece-key/recording.gif',
mime_type: 'image/gif',
})
const records = await run(input)
expect(createReadStream).toHaveBeenCalledTimes(1)
expect(codeToHtml).toHaveBeenCalledExactlyOnceWith(code, {
lang: 'javascript',
defaultColor: false,
themes: { light: 'github-light', dark: 'github-dark' },
})
expect(records).toEqual([
{
transformation: 'highlight',
piece_asset_path: source,
asset_path: null,
mime_type: 'text/html',
is_embedded: 1,
content: html,
},
])
})
test.each(['unregistered', 'ungenerated'])(
'skips a selected %s attachment before file access',
async (missing) => {
const input = makeInput()
if (missing === 'unregistered') input.assetKeyToPath.clear()
else input.priorAssets = []
expect(await run(input)).toEqual([])
expect(createReadStream).not.toHaveBeenCalled()
expect(codeToHtml).not.toHaveBeenCalled()
expect(records).toEqual([])
})
test('reads asset, detects language, and highlights via shiki', async () => {
const pieces = makePieces('console.log("hi")')
const html = '<pre class="shiki">...</pre>'
vi.mocked(getLang).mockReturnValue('javascript')
vi.mocked(codeToHtml).mockResolvedValue(html)
const records = await run({
webPiece: makeWebPiece('{"code": "key"}'),
config: makeConfig(['code']),
outDir: '/out',
pieces,
assetKeyToPath: new Map([['key', 'main.js']]),
logger: makeLogger(),
})
expect(pieces.getPieceAsset).toHaveBeenCalledWith('main.js')
expect(getLang).toHaveBeenCalledWith('main.js')
expect(codeToHtml).toHaveBeenCalledWith('console.log("hi")', {
lang: 'javascript',
defaultColor: false,
themes: { light: 'github-light', dark: 'github-dark' },
})
expect(records).toEqual([
{
transformation: 'highlight',
piece_asset_path: 'main.js',
asset_path: null,
mime_type: 'text/html',
is_embedded: 1,
content: html,
},
])
})
test('skips keys missing from assetKeyToPath without reading or highlighting', async () => {
const pieces = makePieces()
const records = await run({
webPiece: makeWebPiece('{"doc": "missing-key"}'),
config: makeConfig(['doc']),
outDir: '/out',
pieces,
assetKeyToPath: emptyMap,
logger: makeLogger(),
})
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
expect(records).toEqual([])
})
test('falls back to "text" when getLang returns null', async () => {
const pieces = makePieces('plain text')
vi.mocked(getLang).mockReturnValue(null)
vi.mocked(codeToHtml).mockResolvedValue('<pre>...</pre>')
await run({
webPiece: makeWebPiece('{"doc": "key"}'),
config: makeConfig(['doc']),
outDir: '/out',
pieces,
assetKeyToPath: new Map([['key', 'notes.unknown']]),
logger: makeLogger(),
})
expect(codeToHtml).toHaveBeenCalledWith(
'plain text',
expect.objectContaining({ lang: 'text' })
)
})
test('throws on shiki error', async () => {
const pieces = makePieces()
vi.mocked(getLang).mockReturnValue('javascript')
vi.mocked(codeToHtml).mockRejectedValue(new Error('shiki failed'))
await expect(
run({
webPiece: makeWebPiece('{"code": "key"}'),
config: makeConfig(['code']),
outDir: '/out',
pieces,
assetKeyToPath: new Map([['key', 'main.js']]),
logger: makeLogger(),
expect(input.logger.warn).toHaveBeenCalledWith(
'Highlight skipped',
expect.objectContaining({
reason: 'not a registered, generated attachment',
})
).rejects.toThrow('shiki failed')
})
)
}
)
test('produces one record per attachment', async () => {
const pieces = makePieces('content')
vi.mocked(getLang).mockReturnValue('javascript')
vi.mocked(codeToHtml).mockResolvedValue('<pre class="shiki">...</pre>')
test('rejects an oversized source before reading its contents', async () => {
const input = makeInput()
await writeFile(path.join(directory, output), Buffer.alloc(100 * 1024 + 1, 'a'))
const map = new Map([
['key1', 'main.js'],
['key2', 'report.pdf'],
['key3', 'app.ts'],
])
const records = await run({
webPiece: makeWebPiece('{"files": ["key1", "key2", "key3"]}'),
config: makeConfig(['files']),
outDir: '/out',
pieces,
assetKeyToPath: map,
logger: makeLogger(),
})
expect(pieces.getPieceAsset).toHaveBeenCalledTimes(3)
expect(records).toHaveLength(3)
expect(records.map((r) => r.piece_asset_path)).toEqual(['main.js', 'report.pdf', 'app.ts'])
})
test('returns empty array for piece type not in config', async () => {
const pieces = makePieces()
const webPiece = { ...makeWebPiece('{"code": "main.js"}'), type: 'unknown' }
const records = await run({
webPiece,
config: makeConfig(['code']),
outDir: '/out',
pieces,
assetKeyToPath: emptyMap,
logger: makeLogger(),
})
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
expect(records).toEqual([])
expect(await run(input)).toEqual([])
expect(createReadStream).not.toHaveBeenCalled()
expect(codeToHtml).not.toHaveBeenCalled()
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
file: input.webPiece.file_path,
asset: source,
reason: 'source exceeds 102400 bytes',
})
})
test('caps the read and rejects a source that grew after stat', async () => {
const input = makeInput()
const filename = path.join(directory, output)
const staleStat = await stat(filename)
await writeFile(filename, Buffer.alloc(200 * 1024, 'a'))
vi.mocked(stat).mockResolvedValueOnce(staleStat)
expect(await run(input)).toEqual([])
expect(createReadStream).toHaveBeenCalledWith(filename, { start: 0, end: 100 * 1024 })
expect(codeToHtml).not.toHaveBeenCalled()
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
file: input.webPiece.file_path,
asset: source,
reason: 'source exceeds 102400 bytes',
})
})
test.each([
['NUL bytes', Buffer.from('GIF89a\0'), 'source contains NUL bytes'],
['invalid UTF-8', Buffer.from([0xc3, 0x28]), 'source is not valid UTF-8'],
])('rejects %s before Shiki, regardless of the code extension', async (_name, bytes, reason) => {
const input = makeInput()
await writeFile(path.join(directory, output), bytes)
expect(await run(input)).toEqual([])
expect(codeToHtml).not.toHaveBeenCalled()
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
file: input.webPiece.file_path,
asset: source,
reason,
})
})
test('renders selected valid text with an unknown extension as plain text', async () => {
const input = makeInput()
const output = 'snippets/piece-key/notes.unknown'
input.assetKeyToPath.set('code', '.assets/example/notes.unknown')
input.priorAssets = [
{
asset_key: 'code',
transformation: 'attachment',
asset_path: output,
mime_type: 'application/octet-stream',
},
]
await writeFile(path.join(directory, output), 'plain text')
await run(input)
expect(codeToHtml).toHaveBeenCalledWith('plain text', expect.objectContaining({ lang: 'text' }))
})
test('discards generated HTML exceeding the byte limit', async () => {
const input = makeInput()
vi.mocked(codeToHtml).mockResolvedValue('é'.repeat(256 * 1024 + 1))
expect(await run(input)).toEqual([])
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
file: input.webPiece.file_path,
asset: source,
reason: 'generated HTML exceeds 524288 bytes',
})
})
test('does not swallow filesystem failures', async () => {
await rm(path.join(directory, output))
await expect(run(makeInput())).rejects.toMatchObject({ code: 'ENOENT' })
})
test('does not swallow Shiki failures', async () => {
vi.mocked(codeToHtml).mockRejectedValue(new Error('shiki failed'))
await expect(run(makeInput())).rejects.toThrow('shiki failed')
})
@@ -1,59 +1,72 @@
import path from 'node:path'
import { getFrontmatterValues } from '@luzzle/core'
import { codeToHtml } from 'shiki'
import { getLang } from './lang.js'
import { readHighlightSource } from './source.js'
import type { TransformInput, AssetRecord } from '../utils/types.js'
export async function run({
webPiece,
config,
pieces,
assetKeyToPath,
}: TransformInput): Promise<AssetRecord[]> {
const pieceConfig = config.pieces.find((p) => p.type === webPiece.type)
if (!pieceConfig?.fields.attachments) return []
const MAX_HTML_BYTES = 512 * 1024
const frontmatter = JSON.parse(webPiece.json_metadata)
const attachments = pieceConfig.fields.attachments
type HighlightAttachment = { source: string; filename: string }
export async function run(input: TransformInput): Promise<AssetRecord[]> {
const records: AssetRecord[] = []
for (const { source, filename } of selectHighlightAttachments(input)) {
const skip = (reason: string) => warnSkipped(input, source, reason)
const code = await readHighlightSource(filename, skip)
if (code === undefined) continue
for (const field of attachments) {
const values = getFrontmatterValues<string>(frontmatter, field).flat()
const assets = values.reduce(
(maps, key) => {
const path = assetKeyToPath.get(key)
if (path) {
maps.push({ path, key })
}
return maps
},
[] as { path: string; key: string }[]
)
for (const asset of assets) {
const lang = getLang(asset.path) || 'text'
const buffer = await pieces.getPieceAsset(asset.path)
const code = buffer.toString('utf-8')
const html = await codeToHtml(code, {
lang,
defaultColor: false,
themes: {
light: config.theme.markdown.code.light,
dark: config.theme.markdown.code.dark,
},
})
records.push({
transformation: 'highlight',
piece_asset_path: asset.path,
asset_path: null,
mime_type: 'text/html',
is_embedded: 1,
content: html,
})
const lang = getLang(source) || 'text'
const html = await codeToHtml(code, {
lang,
defaultColor: false,
themes: input.config.theme.markdown.code,
})
if (Buffer.byteLength(html) > MAX_HTML_BYTES) {
skip(`generated HTML exceeds ${MAX_HTML_BYTES} bytes`)
continue
}
}
records.push({
transformation: 'highlight',
piece_asset_path: source,
asset_path: null,
mime_type: 'text/html',
is_embedded: 1,
content: html,
})
}
return records
}
function selectHighlightAttachments(input: TransformInput): HighlightAttachment[] {
const { webPiece, config, outDir, assetKeyToPath, priorAssets = [] } = input
const selectors = config.pieces.find((piece) => piece.type === webPiece.type)?.fields.highlight
if (!selectors?.length) return []
const frontmatter = JSON.parse(webPiece.json_metadata)
const selectedKeys = new Set(
selectors.flatMap((field) => getFrontmatterValues<string>(frontmatter, field).flat())
)
const attachments = new Map(
priorAssets
.filter((asset) => asset.transformation === 'attachment')
.map((asset) => [asset.asset_key, asset])
)
const selected: HighlightAttachment[] = []
for (const key of selectedKeys) {
const source = assetKeyToPath.get(key)
const attachment = attachments.get(key)
if (!source || !attachment?.asset_path) {
warnSkipped(input, source || key, 'not a registered, generated attachment')
continue
}
selected.push({ source, filename: path.join(outDir, attachment.asset_path) })
}
return selected
}
function warnSkipped({ webPiece, logger }: TransformInput, asset: string, reason: string): void {
logger.warn('Highlight skipped', { file: webPiece.file_path, asset, reason })
}
@@ -0,0 +1,37 @@
import { isUtf8 } from 'node:buffer'
import { createReadStream } from 'node:fs'
import { stat } from 'node:fs/promises'
const MAX_SOURCE_BYTES = 100 * 1024
export async function readHighlightSource(
filename: string,
skip: (reason: string) => void
): Promise<string | undefined> {
const fileStat = await stat(filename)
if (fileStat.size > MAX_SOURCE_BYTES) {
skip(`source exceeds ${MAX_SOURCE_BYTES} bytes`)
return
}
// Cap the read too, in case the file grows after stat.
const chunks: Buffer[] = []
for await (const chunk of createReadStream(filename, { start: 0, end: MAX_SOURCE_BYTES })) {
chunks.push(chunk)
}
const bytes = Buffer.concat(chunks)
if (bytes.length > MAX_SOURCE_BYTES) {
skip(`source exceeds ${MAX_SOURCE_BYTES} bytes`)
return
}
if (bytes.includes(0)) {
skip('source contains NUL bytes')
return
}
if (!isUtf8(bytes)) {
skip('source is not valid UTF-8')
return
}
return bytes.toString('utf8')
}
@@ -10,6 +10,7 @@ import { previewParseStep, type ParsedPreview } from '../steps/preview-parse.js'
import { getTransforms } from '../transforms/index.js'
import { run as attachment } from '../transforms/attachment.js'
import { run as markdown } from '../transforms/markdown/index.js'
import { run as highlight } from '../transforms/highlight/index.js'
import { generateAssetKey } from '../assets/key.js'
import { completed } from '../core/step.js'
import type { DurableStepApi } from '../core/run-progress-phase.js'
@@ -185,6 +186,45 @@ describe('preview attachment production', () => {
expect(ctx.logger.warn).not.toHaveBeenCalled()
})
test('highlights selected text after attachment generation, leaving binary files downloadable', async () => {
const imagePath = '.assets/diagram.gif'
const imageKey = generateAssetKey(imagePath, ctx.config.assets.salt)
const image = Buffer.from('R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==', 'base64')
await writeFile(path.join(directory, imagePath), image)
parsed.pathToKey.set(imagePath, imageKey)
parsed.keyToPath.set(imageKey, imagePath)
const metadata = JSON.parse(parsed.webPiece.json_metadata)
metadata.files = [{ file: imageKey }, metadata.files[0]]
parsed.webPiece.json_metadata = JSON.stringify(metadata)
ctx.config.pieces[0].fields.highlight = ['files[*].file']
getTransforms().set('highlight', { run: highlight })
const result = await workflow({
input: { filePath: parsed.webPiece.file_path },
run: { id: 'job-highlight' },
step,
})
const highlights = result.transforms.filter((asset) => asset.transformation === 'highlight')
expect(highlights).toHaveLength(1)
expect(highlights[0].piece_asset_path).toBe('.assets/fibonacci.py')
expect(highlights[0].content).toContain('fibonacci')
const download = result.transforms.find(
(asset) => asset.transformation === 'attachment' && asset.asset_key === imageKey
)
expect(download?.mime_type).toBe('image/gif')
expect(
await readFile(
path.join(directory, 'generated/previews/job-highlight', download!.asset_path!)
)
).toEqual(image)
expect(ctx.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
file: parsed.webPiece.file_path,
asset: imagePath,
reason: 'source contains NUL bytes',
})
})
test('does not report a generated attachment when the source cannot be copied', async () => {
await rm(path.join(directory, '.assets/fibonacci.py'))
vi.spyOn(console, 'error').mockImplementation(() => {})