Make attachment highlighting opt-in #199
No files matched your search
@@ -0,0 +1,6 @@
|
||||
---
|
||||
"@luzzle/web.worker": patch
|
||||
"@luzzle/web": patch
|
||||
---
|
||||
|
||||
Require fields.highlight selectors for attachment highlighting. Skip oversized, NUL-containing or invalid UTF-8 sources and oversized generated HTML, leaving original attachments downloadable.
|
||||
@@ -157,6 +157,8 @@ pieces:
|
||||
tags: keywords
|
||||
attachments:
|
||||
- files[*].file
|
||||
highlight:
|
||||
- files[*].file
|
||||
- type: words
|
||||
templates:
|
||||
icon:
|
||||
|
||||
@@ -35,6 +35,22 @@ test("consumers can supply complete extended schemas with their own defaults", (
|
||||
expect(config.pieces[0].badge).toBe("new");
|
||||
});
|
||||
|
||||
test("attachment highlighting requires explicit selectors", () => {
|
||||
const fields = { ...piece.fields, attachments: ["files[*].file"] };
|
||||
const config = validateConfig({ pieces: [{ ...piece, fields }] }, {});
|
||||
expect(config.pieces[0].fields.highlight).toBeUndefined();
|
||||
|
||||
const selected = validateConfig(
|
||||
{
|
||||
pieces: [
|
||||
{ ...piece, fields: { ...fields, highlight: ["files[*].file"] } },
|
||||
],
|
||||
},
|
||||
{},
|
||||
);
|
||||
expect(selected.pieces[0].fields.highlight).toEqual(["files[*].file"]);
|
||||
});
|
||||
|
||||
test("loading defaults preserves the supplied schema and base config contract", () => {
|
||||
const config = loadConfig(undefined, { schema });
|
||||
expect(config.instance).toBe("demo");
|
||||
|
||||
@@ -487,6 +487,13 @@ const PiecesSchema = Type.Array(
|
||||
examples: [["files[*].file"]],
|
||||
}),
|
||||
),
|
||||
highlight: Type.Optional(
|
||||
Type.Array(Type.String(), {
|
||||
description:
|
||||
"Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
|
||||
examples: [["files[*].file"]],
|
||||
}),
|
||||
),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
),
|
||||
|
||||
@@ -598,6 +598,18 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"highlight": {
|
||||
"description": "Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
|
||||
"examples": [
|
||||
[
|
||||
"files[*].file"
|
||||
]
|
||||
],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -598,6 +598,18 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"highlight": {
|
||||
"description": "Frontmatter selectors for published attachments to highlight as code or text. Omit to disable attachment highlighting.",
|
||||
"examples": [
|
||||
[
|
||||
"files[*].file"
|
||||
]
|
||||
],
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -1,199 +1,242 @@
|
||||
import { describe, test, expect, vi, afterEach } from 'vitest'
|
||||
import { run } from './index.js'
|
||||
import { codeToHtml } from 'shiki'
|
||||
import { getLang } from './lang.js'
|
||||
import type { Config } from '@luzzle/web.config'
|
||||
import type { WebPieces } from '../../services/db.js'
|
||||
import { afterEach, beforeEach, expect, test, vi } from 'vitest'
|
||||
import { createReadStream } from 'node:fs'
|
||||
import type * as FileSystem from 'node:fs'
|
||||
import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import type * as FileSystemPromises from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { validateConfig } from '@luzzle/web.config'
|
||||
import type { Pieces } from '@luzzle/core'
|
||||
import { codeToHtml } from 'shiki'
|
||||
import type * as Shiki from 'shiki'
|
||||
import type { TransformInput } from '../utils/types.js'
|
||||
import { makeLogger } from '../../../test/logger.js'
|
||||
import { run } from './index.js'
|
||||
|
||||
vi.mock('shiki', () => ({
|
||||
vi.mock('shiki', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof Shiki>()),
|
||||
codeToHtml: vi.fn(),
|
||||
}))
|
||||
vi.mock('./lang.js', () => ({
|
||||
getLang: vi.fn(),
|
||||
}))
|
||||
|
||||
const emptyMap = new Map<string, string>()
|
||||
|
||||
const makeWebPiece = (json_metadata = '{}'): WebPieces => ({
|
||||
id: '1',
|
||||
type: 'books',
|
||||
date_updated: 100,
|
||||
date_added: 50,
|
||||
json_metadata,
|
||||
file_path: 'book.md',
|
||||
key: 'key123',
|
||||
slug: 'my-book',
|
||||
title: 'My Book',
|
||||
vi.mock('node:fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof FileSystem>()
|
||||
return { ...actual, createReadStream: vi.fn(actual.createReadStream) }
|
||||
})
|
||||
vi.mock('node:fs/promises', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof FileSystemPromises>()
|
||||
return { ...actual, stat: vi.fn(actual.stat) }
|
||||
})
|
||||
|
||||
const makeConfig = (attachments?: string[]): Config =>
|
||||
({
|
||||
url: { app: 'http://localhost' },
|
||||
theme: { markdown: { code: { light: 'github-light', dark: 'github-dark' } } },
|
||||
pieces: [
|
||||
const source = '.assets/example/main.js'
|
||||
const output = 'snippets/piece-key/main.js'
|
||||
const code = 'console.log("hi")'
|
||||
const html = '<pre class="shiki">highlighted code</pre>'
|
||||
let directory: string
|
||||
|
||||
function makeInput(): TransformInput {
|
||||
return {
|
||||
webPiece: {
|
||||
id: 'example',
|
||||
key: 'piece-key',
|
||||
type: 'snippets',
|
||||
slug: 'example',
|
||||
title: 'Example',
|
||||
file_path: 'example.snippets.md',
|
||||
date_added: 1,
|
||||
json_metadata: JSON.stringify({ files: [{ file: 'code' }] }),
|
||||
},
|
||||
config: validateConfig(
|
||||
{
|
||||
type: 'books',
|
||||
fields: {
|
||||
title: 'title',
|
||||
date_consumed: 'date_consumed',
|
||||
...(attachments ? { attachments } : {}),
|
||||
},
|
||||
pieces: [
|
||||
{
|
||||
type: 'snippets',
|
||||
fields: {
|
||||
title: 'title',
|
||||
date_consumed: 'date',
|
||||
attachments: ['files[*].file', 'downloads'],
|
||||
highlight: ['files[*].file'],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{}
|
||||
),
|
||||
outDir: directory,
|
||||
pieces: {} as Pieces,
|
||||
assetKeyToPath: new Map([['code', source]]),
|
||||
priorAssets: [
|
||||
{
|
||||
asset_key: 'code',
|
||||
transformation: 'attachment',
|
||||
asset_path: output,
|
||||
mime_type: 'text/javascript',
|
||||
},
|
||||
],
|
||||
}) as unknown as Config
|
||||
logger: makeLogger(),
|
||||
}
|
||||
}
|
||||
|
||||
const makePieces = (content: string = 'console.log("hi")') =>
|
||||
({ getPieceAsset: vi.fn().mockResolvedValue(Buffer.from(content)) }) as unknown as Pieces
|
||||
beforeEach(async () => {
|
||||
directory = await mkdtemp(path.join(tmpdir(), 'luzzle-highlight-'))
|
||||
await mkdir(path.dirname(path.join(directory, output)), { recursive: true })
|
||||
await writeFile(path.join(directory, output), code)
|
||||
vi.mocked(codeToHtml).mockResolvedValue(html)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
afterEach(async () => {
|
||||
vi.clearAllMocks()
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('transforms/highlight', () => {
|
||||
test('returns empty array when piece type has no attachments config', async () => {
|
||||
const pieces = makePieces()
|
||||
test('does not read or highlight attachments without explicit selection', async () => {
|
||||
const input = makeInput()
|
||||
delete input.config.pieces[0].fields.highlight
|
||||
|
||||
const records = await run({
|
||||
webPiece: makeWebPiece('{}'),
|
||||
config: makeConfig(),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: emptyMap,
|
||||
logger: makeLogger(),
|
||||
})
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(createReadStream).not.toHaveBeenCalled()
|
||||
expect(codeToHtml).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
|
||||
test('highlights selected originals once, leaving other attachments unread', async () => {
|
||||
const input = makeInput()
|
||||
input.config.pieces[0].fields.highlight?.push('source')
|
||||
input.webPiece.json_metadata = JSON.stringify({
|
||||
files: [{ file: 'code' }],
|
||||
source: 'code',
|
||||
downloads: ['recording'],
|
||||
})
|
||||
input.assetKeyToPath.set('recording', '.assets/example/recording.gif')
|
||||
input.priorAssets?.push({
|
||||
asset_key: 'recording',
|
||||
transformation: 'attachment',
|
||||
asset_path: 'snippets/piece-key/recording.gif',
|
||||
mime_type: 'image/gif',
|
||||
})
|
||||
|
||||
const records = await run(input)
|
||||
|
||||
expect(createReadStream).toHaveBeenCalledTimes(1)
|
||||
expect(codeToHtml).toHaveBeenCalledExactlyOnceWith(code, {
|
||||
lang: 'javascript',
|
||||
defaultColor: false,
|
||||
themes: { light: 'github-light', dark: 'github-dark' },
|
||||
})
|
||||
expect(records).toEqual([
|
||||
{
|
||||
transformation: 'highlight',
|
||||
piece_asset_path: source,
|
||||
asset_path: null,
|
||||
mime_type: 'text/html',
|
||||
is_embedded: 1,
|
||||
content: html,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test.each(['unregistered', 'ungenerated'])(
|
||||
'skips a selected %s attachment before file access',
|
||||
async (missing) => {
|
||||
const input = makeInput()
|
||||
if (missing === 'unregistered') input.assetKeyToPath.clear()
|
||||
else input.priorAssets = []
|
||||
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(createReadStream).not.toHaveBeenCalled()
|
||||
expect(codeToHtml).not.toHaveBeenCalled()
|
||||
expect(records).toEqual([])
|
||||
})
|
||||
|
||||
test('reads asset, detects language, and highlights via shiki', async () => {
|
||||
const pieces = makePieces('console.log("hi")')
|
||||
const html = '<pre class="shiki">...</pre>'
|
||||
vi.mocked(getLang).mockReturnValue('javascript')
|
||||
vi.mocked(codeToHtml).mockResolvedValue(html)
|
||||
|
||||
const records = await run({
|
||||
webPiece: makeWebPiece('{"code": "key"}'),
|
||||
config: makeConfig(['code']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: new Map([['key', 'main.js']]),
|
||||
logger: makeLogger(),
|
||||
})
|
||||
|
||||
expect(pieces.getPieceAsset).toHaveBeenCalledWith('main.js')
|
||||
expect(getLang).toHaveBeenCalledWith('main.js')
|
||||
expect(codeToHtml).toHaveBeenCalledWith('console.log("hi")', {
|
||||
lang: 'javascript',
|
||||
defaultColor: false,
|
||||
themes: { light: 'github-light', dark: 'github-dark' },
|
||||
})
|
||||
expect(records).toEqual([
|
||||
{
|
||||
transformation: 'highlight',
|
||||
piece_asset_path: 'main.js',
|
||||
asset_path: null,
|
||||
mime_type: 'text/html',
|
||||
is_embedded: 1,
|
||||
content: html,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
test('skips keys missing from assetKeyToPath without reading or highlighting', async () => {
|
||||
const pieces = makePieces()
|
||||
|
||||
const records = await run({
|
||||
webPiece: makeWebPiece('{"doc": "missing-key"}'),
|
||||
config: makeConfig(['doc']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: emptyMap,
|
||||
logger: makeLogger(),
|
||||
})
|
||||
|
||||
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
|
||||
expect(records).toEqual([])
|
||||
})
|
||||
|
||||
test('falls back to "text" when getLang returns null', async () => {
|
||||
const pieces = makePieces('plain text')
|
||||
vi.mocked(getLang).mockReturnValue(null)
|
||||
vi.mocked(codeToHtml).mockResolvedValue('<pre>...</pre>')
|
||||
|
||||
await run({
|
||||
webPiece: makeWebPiece('{"doc": "key"}'),
|
||||
config: makeConfig(['doc']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: new Map([['key', 'notes.unknown']]),
|
||||
logger: makeLogger(),
|
||||
})
|
||||
|
||||
expect(codeToHtml).toHaveBeenCalledWith(
|
||||
'plain text',
|
||||
expect.objectContaining({ lang: 'text' })
|
||||
)
|
||||
})
|
||||
|
||||
test('throws on shiki error', async () => {
|
||||
const pieces = makePieces()
|
||||
vi.mocked(getLang).mockReturnValue('javascript')
|
||||
vi.mocked(codeToHtml).mockRejectedValue(new Error('shiki failed'))
|
||||
|
||||
await expect(
|
||||
run({
|
||||
webPiece: makeWebPiece('{"code": "key"}'),
|
||||
config: makeConfig(['code']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: new Map([['key', 'main.js']]),
|
||||
logger: makeLogger(),
|
||||
expect(input.logger.warn).toHaveBeenCalledWith(
|
||||
'Highlight skipped',
|
||||
expect.objectContaining({
|
||||
reason: 'not a registered, generated attachment',
|
||||
})
|
||||
).rejects.toThrow('shiki failed')
|
||||
})
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
test('produces one record per attachment', async () => {
|
||||
const pieces = makePieces('content')
|
||||
vi.mocked(getLang).mockReturnValue('javascript')
|
||||
vi.mocked(codeToHtml).mockResolvedValue('<pre class="shiki">...</pre>')
|
||||
test('rejects an oversized source before reading its contents', async () => {
|
||||
const input = makeInput()
|
||||
await writeFile(path.join(directory, output), Buffer.alloc(100 * 1024 + 1, 'a'))
|
||||
|
||||
const map = new Map([
|
||||
['key1', 'main.js'],
|
||||
['key2', 'report.pdf'],
|
||||
['key3', 'app.ts'],
|
||||
])
|
||||
const records = await run({
|
||||
webPiece: makeWebPiece('{"files": ["key1", "key2", "key3"]}'),
|
||||
config: makeConfig(['files']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: map,
|
||||
logger: makeLogger(),
|
||||
})
|
||||
|
||||
expect(pieces.getPieceAsset).toHaveBeenCalledTimes(3)
|
||||
expect(records).toHaveLength(3)
|
||||
expect(records.map((r) => r.piece_asset_path)).toEqual(['main.js', 'report.pdf', 'app.ts'])
|
||||
})
|
||||
|
||||
test('returns empty array for piece type not in config', async () => {
|
||||
const pieces = makePieces()
|
||||
const webPiece = { ...makeWebPiece('{"code": "main.js"}'), type: 'unknown' }
|
||||
|
||||
const records = await run({
|
||||
webPiece,
|
||||
config: makeConfig(['code']),
|
||||
outDir: '/out',
|
||||
pieces,
|
||||
assetKeyToPath: emptyMap,
|
||||
logger: makeLogger(),
|
||||
})
|
||||
|
||||
expect(pieces.getPieceAsset).not.toHaveBeenCalled()
|
||||
expect(records).toEqual([])
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(createReadStream).not.toHaveBeenCalled()
|
||||
expect(codeToHtml).not.toHaveBeenCalled()
|
||||
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
|
||||
file: input.webPiece.file_path,
|
||||
asset: source,
|
||||
reason: 'source exceeds 102400 bytes',
|
||||
})
|
||||
})
|
||||
|
||||
test('caps the read and rejects a source that grew after stat', async () => {
|
||||
const input = makeInput()
|
||||
const filename = path.join(directory, output)
|
||||
const staleStat = await stat(filename)
|
||||
await writeFile(filename, Buffer.alloc(200 * 1024, 'a'))
|
||||
vi.mocked(stat).mockResolvedValueOnce(staleStat)
|
||||
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(createReadStream).toHaveBeenCalledWith(filename, { start: 0, end: 100 * 1024 })
|
||||
expect(codeToHtml).not.toHaveBeenCalled()
|
||||
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
|
||||
file: input.webPiece.file_path,
|
||||
asset: source,
|
||||
reason: 'source exceeds 102400 bytes',
|
||||
})
|
||||
})
|
||||
|
||||
test.each([
|
||||
['NUL bytes', Buffer.from('GIF89a\0'), 'source contains NUL bytes'],
|
||||
['invalid UTF-8', Buffer.from([0xc3, 0x28]), 'source is not valid UTF-8'],
|
||||
])('rejects %s before Shiki, regardless of the code extension', async (_name, bytes, reason) => {
|
||||
const input = makeInput()
|
||||
await writeFile(path.join(directory, output), bytes)
|
||||
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(codeToHtml).not.toHaveBeenCalled()
|
||||
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
|
||||
file: input.webPiece.file_path,
|
||||
asset: source,
|
||||
reason,
|
||||
})
|
||||
})
|
||||
|
||||
test('renders selected valid text with an unknown extension as plain text', async () => {
|
||||
const input = makeInput()
|
||||
const output = 'snippets/piece-key/notes.unknown'
|
||||
input.assetKeyToPath.set('code', '.assets/example/notes.unknown')
|
||||
input.priorAssets = [
|
||||
{
|
||||
asset_key: 'code',
|
||||
transformation: 'attachment',
|
||||
asset_path: output,
|
||||
mime_type: 'application/octet-stream',
|
||||
},
|
||||
]
|
||||
await writeFile(path.join(directory, output), 'plain text')
|
||||
|
||||
await run(input)
|
||||
|
||||
expect(codeToHtml).toHaveBeenCalledWith('plain text', expect.objectContaining({ lang: 'text' }))
|
||||
})
|
||||
|
||||
test('discards generated HTML exceeding the byte limit', async () => {
|
||||
const input = makeInput()
|
||||
vi.mocked(codeToHtml).mockResolvedValue('é'.repeat(256 * 1024 + 1))
|
||||
|
||||
expect(await run(input)).toEqual([])
|
||||
expect(input.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
|
||||
file: input.webPiece.file_path,
|
||||
asset: source,
|
||||
reason: 'generated HTML exceeds 524288 bytes',
|
||||
})
|
||||
})
|
||||
|
||||
test('does not swallow filesystem failures', async () => {
|
||||
await rm(path.join(directory, output))
|
||||
|
||||
await expect(run(makeInput())).rejects.toMatchObject({ code: 'ENOENT' })
|
||||
})
|
||||
|
||||
test('does not swallow Shiki failures', async () => {
|
||||
vi.mocked(codeToHtml).mockRejectedValue(new Error('shiki failed'))
|
||||
|
||||
await expect(run(makeInput())).rejects.toThrow('shiki failed')
|
||||
})
|
||||
@@ -1,59 +1,72 @@
|
||||
import path from 'node:path'
|
||||
import { getFrontmatterValues } from '@luzzle/core'
|
||||
import { codeToHtml } from 'shiki'
|
||||
import { getLang } from './lang.js'
|
||||
import { readHighlightSource } from './source.js'
|
||||
import type { TransformInput, AssetRecord } from '../utils/types.js'
|
||||
|
||||
export async function run({
|
||||
webPiece,
|
||||
config,
|
||||
pieces,
|
||||
assetKeyToPath,
|
||||
}: TransformInput): Promise<AssetRecord[]> {
|
||||
const pieceConfig = config.pieces.find((p) => p.type === webPiece.type)
|
||||
if (!pieceConfig?.fields.attachments) return []
|
||||
const MAX_HTML_BYTES = 512 * 1024
|
||||
|
||||
const frontmatter = JSON.parse(webPiece.json_metadata)
|
||||
const attachments = pieceConfig.fields.attachments
|
||||
type HighlightAttachment = { source: string; filename: string }
|
||||
|
||||
export async function run(input: TransformInput): Promise<AssetRecord[]> {
|
||||
const records: AssetRecord[] = []
|
||||
for (const { source, filename } of selectHighlightAttachments(input)) {
|
||||
const skip = (reason: string) => warnSkipped(input, source, reason)
|
||||
const code = await readHighlightSource(filename, skip)
|
||||
if (code === undefined) continue
|
||||
|
||||
for (const field of attachments) {
|
||||
const values = getFrontmatterValues<string>(frontmatter, field).flat()
|
||||
|
||||
const assets = values.reduce(
|
||||
(maps, key) => {
|
||||
const path = assetKeyToPath.get(key)
|
||||
if (path) {
|
||||
maps.push({ path, key })
|
||||
}
|
||||
return maps
|
||||
},
|
||||
[] as { path: string; key: string }[]
|
||||
)
|
||||
|
||||
for (const asset of assets) {
|
||||
const lang = getLang(asset.path) || 'text'
|
||||
const buffer = await pieces.getPieceAsset(asset.path)
|
||||
const code = buffer.toString('utf-8')
|
||||
|
||||
const html = await codeToHtml(code, {
|
||||
lang,
|
||||
defaultColor: false,
|
||||
themes: {
|
||||
light: config.theme.markdown.code.light,
|
||||
dark: config.theme.markdown.code.dark,
|
||||
},
|
||||
})
|
||||
|
||||
records.push({
|
||||
transformation: 'highlight',
|
||||
piece_asset_path: asset.path,
|
||||
asset_path: null,
|
||||
mime_type: 'text/html',
|
||||
is_embedded: 1,
|
||||
content: html,
|
||||
})
|
||||
const lang = getLang(source) || 'text'
|
||||
const html = await codeToHtml(code, {
|
||||
lang,
|
||||
defaultColor: false,
|
||||
themes: input.config.theme.markdown.code,
|
||||
})
|
||||
if (Buffer.byteLength(html) > MAX_HTML_BYTES) {
|
||||
skip(`generated HTML exceeds ${MAX_HTML_BYTES} bytes`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
records.push({
|
||||
transformation: 'highlight',
|
||||
piece_asset_path: source,
|
||||
asset_path: null,
|
||||
mime_type: 'text/html',
|
||||
is_embedded: 1,
|
||||
content: html,
|
||||
})
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
function selectHighlightAttachments(input: TransformInput): HighlightAttachment[] {
|
||||
const { webPiece, config, outDir, assetKeyToPath, priorAssets = [] } = input
|
||||
const selectors = config.pieces.find((piece) => piece.type === webPiece.type)?.fields.highlight
|
||||
if (!selectors?.length) return []
|
||||
|
||||
const frontmatter = JSON.parse(webPiece.json_metadata)
|
||||
const selectedKeys = new Set(
|
||||
selectors.flatMap((field) => getFrontmatterValues<string>(frontmatter, field).flat())
|
||||
)
|
||||
const attachments = new Map(
|
||||
priorAssets
|
||||
.filter((asset) => asset.transformation === 'attachment')
|
||||
.map((asset) => [asset.asset_key, asset])
|
||||
)
|
||||
const selected: HighlightAttachment[] = []
|
||||
|
||||
for (const key of selectedKeys) {
|
||||
const source = assetKeyToPath.get(key)
|
||||
const attachment = attachments.get(key)
|
||||
if (!source || !attachment?.asset_path) {
|
||||
warnSkipped(input, source || key, 'not a registered, generated attachment')
|
||||
continue
|
||||
}
|
||||
selected.push({ source, filename: path.join(outDir, attachment.asset_path) })
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
function warnSkipped({ webPiece, logger }: TransformInput, asset: string, reason: string): void {
|
||||
logger.warn('Highlight skipped', { file: webPiece.file_path, asset, reason })
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { isUtf8 } from 'node:buffer'
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { stat } from 'node:fs/promises'
|
||||
|
||||
const MAX_SOURCE_BYTES = 100 * 1024
|
||||
|
||||
export async function readHighlightSource(
|
||||
filename: string,
|
||||
skip: (reason: string) => void
|
||||
): Promise<string | undefined> {
|
||||
const fileStat = await stat(filename)
|
||||
if (fileStat.size > MAX_SOURCE_BYTES) {
|
||||
skip(`source exceeds ${MAX_SOURCE_BYTES} bytes`)
|
||||
return
|
||||
}
|
||||
|
||||
// Cap the read too, in case the file grows after stat.
|
||||
const chunks: Buffer[] = []
|
||||
for await (const chunk of createReadStream(filename, { start: 0, end: MAX_SOURCE_BYTES })) {
|
||||
chunks.push(chunk)
|
||||
}
|
||||
const bytes = Buffer.concat(chunks)
|
||||
if (bytes.length > MAX_SOURCE_BYTES) {
|
||||
skip(`source exceeds ${MAX_SOURCE_BYTES} bytes`)
|
||||
return
|
||||
}
|
||||
if (bytes.includes(0)) {
|
||||
skip('source contains NUL bytes')
|
||||
return
|
||||
}
|
||||
if (!isUtf8(bytes)) {
|
||||
skip('source is not valid UTF-8')
|
||||
return
|
||||
}
|
||||
|
||||
return bytes.toString('utf8')
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { previewParseStep, type ParsedPreview } from '../steps/preview-parse.js'
|
||||
import { getTransforms } from '../transforms/index.js'
|
||||
import { run as attachment } from '../transforms/attachment.js'
|
||||
import { run as markdown } from '../transforms/markdown/index.js'
|
||||
import { run as highlight } from '../transforms/highlight/index.js'
|
||||
import { generateAssetKey } from '../assets/key.js'
|
||||
import { completed } from '../core/step.js'
|
||||
import type { DurableStepApi } from '../core/run-progress-phase.js'
|
||||
@@ -185,6 +186,45 @@ describe('preview attachment production', () => {
|
||||
expect(ctx.logger.warn).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
test('highlights selected text after attachment generation, leaving binary files downloadable', async () => {
|
||||
const imagePath = '.assets/diagram.gif'
|
||||
const imageKey = generateAssetKey(imagePath, ctx.config.assets.salt)
|
||||
const image = Buffer.from('R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==', 'base64')
|
||||
await writeFile(path.join(directory, imagePath), image)
|
||||
parsed.pathToKey.set(imagePath, imageKey)
|
||||
parsed.keyToPath.set(imageKey, imagePath)
|
||||
const metadata = JSON.parse(parsed.webPiece.json_metadata)
|
||||
metadata.files = [{ file: imageKey }, metadata.files[0]]
|
||||
parsed.webPiece.json_metadata = JSON.stringify(metadata)
|
||||
ctx.config.pieces[0].fields.highlight = ['files[*].file']
|
||||
getTransforms().set('highlight', { run: highlight })
|
||||
|
||||
const result = await workflow({
|
||||
input: { filePath: parsed.webPiece.file_path },
|
||||
run: { id: 'job-highlight' },
|
||||
step,
|
||||
})
|
||||
|
||||
const highlights = result.transforms.filter((asset) => asset.transformation === 'highlight')
|
||||
expect(highlights).toHaveLength(1)
|
||||
expect(highlights[0].piece_asset_path).toBe('.assets/fibonacci.py')
|
||||
expect(highlights[0].content).toContain('fibonacci')
|
||||
const download = result.transforms.find(
|
||||
(asset) => asset.transformation === 'attachment' && asset.asset_key === imageKey
|
||||
)
|
||||
expect(download?.mime_type).toBe('image/gif')
|
||||
expect(
|
||||
await readFile(
|
||||
path.join(directory, 'generated/previews/job-highlight', download!.asset_path!)
|
||||
)
|
||||
).toEqual(image)
|
||||
expect(ctx.logger.warn).toHaveBeenCalledWith('Highlight skipped', {
|
||||
file: parsed.webPiece.file_path,
|
||||
asset: imagePath,
|
||||
reason: 'source contains NUL bytes',
|
||||
})
|
||||
})
|
||||
|
||||
test('does not report a generated attachment when the source cannot be copied', async () => {
|
||||
await rm(path.join(directory, '.assets/fibonacci.py'))
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
|
||||
Reference in new issue
Block a user