diff --git a/apps/cli/src/lib/commands/command/assistant.test.ts b/apps/cli/src/lib/commands/command/assistant.test.ts index cdc9d759..afa824f1 100644 --- a/apps/cli/src/lib/commands/command/assistant.test.ts +++ b/apps/cli/src/lib/commands/command/assistant.test.ts @@ -88,7 +88,11 @@ describe('lib/commands/assistant.ts', () => { const creating = command.run(ctx, { prompt, update, directory } as Arguments) - await expect(creating).rejects.toThrow() + await expect(creating).rejects.toThrow('update and directory are mutually exclusive') + expect(mocks.parseArgs).not.toHaveBeenCalled() + expect(mocks.generatePieceFrontmatter).not.toHaveBeenCalled() + expect(mocks.yamlStringify).not.toHaveBeenCalled() + expect(mocks.consoleLog).not.toHaveBeenCalled() }) test('must use update and title exclusively', async () => { @@ -111,7 +115,11 @@ describe('lib/commands/assistant.ts', () => { const creating = command.run(ctx, { prompt, update, title } as Arguments) - await expect(creating).rejects.toThrow() + await expect(creating).rejects.toThrow('title is only to be used when creating a new piece') + expect(mocks.parseArgs).not.toHaveBeenCalled() + expect(mocks.generatePieceFrontmatter).not.toHaveBeenCalled() + expect(mocks.yamlStringify).not.toHaveBeenCalled() + expect(mocks.consoleLog).not.toHaveBeenCalled() }) test('updates a piece', async () => { @@ -164,7 +172,11 @@ describe('lib/commands/assistant.ts', () => { const creating = command.run(ctx, { prompt, directory } as Arguments) - await expect(creating).rejects.toThrow() + await expect(creating).rejects.toThrow('title is required when creating a new piece') + expect(mocks.parseArgs).not.toHaveBeenCalled() + expect(mocks.generatePieceFrontmatter).not.toHaveBeenCalled() + expect(mocks.yamlStringify).not.toHaveBeenCalled() + expect(mocks.consoleLog).not.toHaveBeenCalled() }) test('creates a new piece', async () => { diff --git a/apps/cli/src/lib/commands/command/assistant.ts b/apps/cli/src/lib/commands/command/assistant.ts index 89a82583..7092994e 100644 --- a/apps/cli/src/lib/commands/command/assistant.ts +++ b/apps/cli/src/lib/commands/command/assistant.ts @@ -52,9 +52,6 @@ const command: Command = { run: async function(ctx, args) { const { prompt, file, update, directory, title } = args - const { piece } = await parsePieceOptionArgv(ctx, args) - const apiKey = ctx.config.get('api_keys.google', '') - const metadata = await pieceFrontMatterFromPrompt(apiKey, piece.schema, prompt, file) if (update && directory) { throw new Error(`update and directory are mutually exclusive`) @@ -68,6 +65,10 @@ const command: Command = { throw new Error(`title is required when creating a new piece`) } + const { piece } = await parsePieceOptionArgv(ctx, args) + const apiKey = ctx.config.get('api_keys.google', '') + const metadata = await pieceFrontMatterFromPrompt(apiKey, piece.schema, prompt, file) + if (update) { const markdown = await piece.get(update) const markdownUpdate = await piece.setFields(markdown, metadata) diff --git a/apps/cli/src/lib/commands/utils/context.fixtures.ts b/apps/cli/src/lib/commands/utils/context.fixtures.ts index 6b9af73d..3ce3a7fa 100644 --- a/apps/cli/src/lib/commands/utils/context.fixtures.ts +++ b/apps/cli/src/lib/commands/utils/context.fixtures.ts @@ -42,6 +42,7 @@ function makeStorage(root: string): LuzzleStorage { parseArgPath: vi.fn(), readFile: vi.fn(), writeFile: vi.fn(), + createFile: vi.fn(), getFilesIn: vi.fn(), exists: vi.fn(), delete: vi.fn(), diff --git a/apps/core/package.json b/apps/core/package.json index c1960d5d..06db1db0 100644 --- a/apps/core/package.json +++ b/apps/core/package.json @@ -48,13 +48,14 @@ "vitest": "catalog:" }, "dependencies": { - "@google/genai": "1.30.0", + "@google/genai": "2.24.0", "@paralleldrive/cuid2": "2.2.2", "@sindresorhus/slugify": "3.0.0", "ajv": "8.17.1", "fdir": "6.5.0", "file-type": "21.3.0", "got": "14.6.5", + "json-schema-traverse": "1.0.0", "jsonpathly": "3.0.0", "kysely": "0.28.17", "remark": "15.0.1", diff --git a/apps/core/src/index.test.ts b/apps/core/src/index.test.ts index dc8032a1..417073a6 100644 --- a/apps/core/src/index.test.ts +++ b/apps/core/src/index.test.ts @@ -7,4 +7,19 @@ describe('index.ts', () => { test('schema', () => { expect(index).toBeDefined() }) + + test('preserves the public generation exports without exposing provider internals', () => { + for (const name of [ + 'generatePieceMetadata', + 'generateFieldValue', + 'generateBody', + 'pieceFrontMatterFromPrompt', + 'validateApiKey', + ] as const) { + expect(index[name]).toBeTypeOf('function') + } + expect(index.DEFAULT_GENERATION_LIMITS).toBeDefined() + expect(Object.hasOwn(index, 'getClient')).toBe(false) + expect(Object.hasOwn(index, 'runGeneration')).toBe(false) + }) }) diff --git a/apps/core/src/index.ts b/apps/core/src/index.ts index ff747cbe..8975fb19 100644 --- a/apps/core/src/index.ts +++ b/apps/core/src/index.ts @@ -17,7 +17,7 @@ import { export * from './database/tables/index.js' export * from './pieces/index.js' -export * from './llm/google.js' +export * from './llm/google/index.js' export * from './storage/index.js' export { diff --git a/apps/core/src/llm/field-schema.test.ts b/apps/core/src/llm/field-schema.test.ts new file mode 100644 index 00000000..6b754225 --- /dev/null +++ b/apps/core/src/llm/field-schema.test.ts @@ -0,0 +1,119 @@ +import type { JSONSchemaType, SchemaObject } from 'ajv' +import { describe, expect, test } from 'vitest' +import compile from '../lib/ajv.js' +import type { PieceFrontmatter } from '../pieces/utils/frontmatter.js' +import { selectFieldSchema } from './field-schema.js' + +const makeSchema = (field: unknown, extra: SchemaObject = {}) => + ({ + type: 'object', + properties: { field, other: { type: 'string' } }, + required: ['other'], + ...extra, + }) as JSONSchemaType + +describe('selectFieldSchema', () => { + test('requires only the selected property and rejects other output keys', () => { + const source = makeSchema( + { type: 'string', minLength: 2 }, + { + minProperties: 2, + allOf: [{ required: ['other'] }], + $defs: { unused: { type: 'integer' } }, + } + ) + const result = selectFieldSchema(source, 'field') + expect(result).toEqual({ + type: 'object', + properties: { field: { type: 'string', minLength: 2 } }, + required: ['field'], + additionalProperties: false, + }) + const validate = compile(result) + expect(validate({ field: 'value' })).toBe(true) + for (const invalid of [{}, { field: 'x' }, { field: 'value', other: 'extra' }]) { + expect(validate(invalid)).toBe(false) + } + }) + + test('preserves inline array/object constraints, nullability, formats, and source data', () => { + const source = makeSchema({ + type: 'array', + items: { + type: 'object', + properties: { date: { type: 'string', nullable: true, format: 'date' } }, + required: ['date'], + }, + }) + const original = structuredClone(source) + const result = selectFieldSchema(source, 'field') + const validate = compile(result) + expect(validate({ field: [{ date: null }, { date: '2026-01-02' }] })).toBe(true) + expect(validate({ field: [{ date: 'invalid' }] })).toBe(false) + expect(source).toEqual(original) + expect(result.properties.field).toEqual(source.properties.field) + expect(result.properties.field).not.toBe(source.properties.field) + }) + + test('does not mistake literal example data or property names for schema references', () => { + const data = { $ref: '#', $dynamicRef: '#example' } + const source = makeSchema({ + type: 'object', + default: data, + examples: [data], + const: data, + enum: [data], + properties: { $ref: { type: 'string' }, $dynamicRef: { type: 'string' } }, + }) + const result = selectFieldSchema(source, 'field') + expect(result.properties.field).toEqual(source.properties.field) + ;(result.properties.field as SchemaObject).default.$ref = 'changed' + expect(data.$ref).toBe('#') + }) + + test('selects literal own keys, not inherited or missing keys', () => { + const source = makeSchema(true, { properties: { 'a/b~c': { type: 'string' } } }) + expect(selectFieldSchema(source, 'a/b~c').required).toEqual(['a/b~c']) + for (const key of ['missing', 'toString', '__proto__']) { + expect(() => selectFieldSchema(source, key)).toThrow('own top-level property') + } + expect(() => + selectFieldSchema({ type: 'object' } as JSONSchemaType, 'field') + ).toThrow('own top-level property') + }) + + test('rejects a reserved field name that the validator skips', () => { + const source = makeSchema(true, { properties: JSON.parse('{"__proto__":{"type":"string"}}') }) + expect(() => selectFieldSchema(source, '__proto__')).toThrow('validator does not support') + }) + + test.each(['$ref', '$dynamicRef', '$recursiveRef'])( + 'rejects an unexpected %s instead of resolving it', + (keyword) => { + expect(() => selectFieldSchema(makeSchema({ [keyword]: '#' }), 'field')).toThrow( + 'reference-free schemas' + ) + } + ) + + test.each([ + makeSchema({ type: 'array', items: { $ref: '#/$defs/item' } }), + makeSchema({ + type: 'object', + allOf: [{ type: 'object' }], + properties: { default: { $ref: '#' } }, + }), + makeSchema({ type: 'string' }, { definitions: { unused: { $ref: '#' } } }), + makeSchema({ type: 'string' }, { $defs: { unused: { $ref: '#' } } }), + makeSchema({ type: 'string' }, { $ref: 'external.json' }), + ])('rejects references anywhere in the input schema', (source) => { + expect(() => selectFieldSchema(source, 'field')).toThrow('reference-free schemas') + }) + + test('preserves the dialect for the normal Ajv compiler', () => { + const dialect = 'http://json-schema.org/draft-07/schema#' + const result = selectFieldSchema(makeSchema({ type: 'string' }, { $schema: dialect }), 'field') + expect(result.$schema).toBe(dialect) + expect(compile(result)({ field: 'value' })).toBe(true) + }) +}) diff --git a/apps/core/src/llm/field-schema.ts b/apps/core/src/llm/field-schema.ts new file mode 100644 index 00000000..60d170e2 --- /dev/null +++ b/apps/core/src/llm/field-schema.ts @@ -0,0 +1,33 @@ +import type { SchemaObject } from 'ajv' +import traverse from 'json-schema-traverse' +import type { PieceFrontmatter, PieceFrontmatterSchema } from '../pieces/utils/frontmatter.js' + +export function selectFieldSchema( + schema: PieceFrontmatterSchema, + key: string +): PieceFrontmatterSchema { + if (!schema.properties || !Object.hasOwn(schema.properties, key)) { + throw new Error('Field generation requires an own top-level property.') + } + if (key === '__proto__') + throw new Error('The validator does not support a field named __proto__.') + + traverse(schema, { + cb(node) { + if ( + ['$ref', '$dynamicRef', '$recursiveRef'].some((keyword) => Object.hasOwn(node, keyword)) + ) { + throw new Error('Field generation requires reference-free schemas.') + } + }, + }) + + const responseSchema: SchemaObject = { + type: 'object', + properties: { [key]: schema.properties[key] }, + required: [key], + additionalProperties: false, + } + if (Object.hasOwn(schema, '$schema')) responseSchema.$schema = schema.$schema + return structuredClone(responseSchema) as PieceFrontmatterSchema +} diff --git a/apps/core/src/llm/google.test.ts b/apps/core/src/llm/google.test.ts deleted file mode 100644 index 5b42e33e..00000000 --- a/apps/core/src/llm/google.test.ts +++ /dev/null @@ -1,390 +0,0 @@ -import type { MockInstance } from 'vitest'; -import { describe, expect, vi, afterEach, test } from 'vitest' -import { pieceFrontMatterFromPrompt, validateApiKey } from './google.js' -import type { - Part, - GenerateContentResponse, - FileState} from '@google/genai'; -import { - GoogleGenAI, - createPartFromUri -} from '@google/genai' -import { fileTypeFromBuffer, fileTypeFromFile } from 'file-type' -import { readFile } from 'fs/promises' -import type { JSONSchemaType } from 'ajv' -import type { PieceFrontmatter } from '../pieces/index.js' - -// Mock schema created locally to remove dependency on piece.fixtures.ts -const makeSchema = ( - name: string -): JSONSchemaType => { - return { - type: 'object', - title: name, - properties: { - title: { type: 'string', examples: ['title'] }, - keywords: { type: 'string', nullable: true }, - }, - required: ['title'], - additionalProperties: false, - } as unknown as JSONSchemaType -} - -vi.mock('@luzzle/core') -vi.mock('file-type') -vi.mock('fs/promises') -vi.mock('@google/genai', () => { - const Gemini = vi.fn() - - Gemini.prototype.models = vi.fn() - Gemini.prototype.files = vi.fn() - Gemini.prototype.models.generateContent = vi.fn() - Gemini.prototype.models.list = vi.fn() - Gemini.prototype.files.upload = vi.fn() - Gemini.prototype.files.get = vi.fn() - - return { - GoogleGenAI: Gemini, - HarmCategory: {}, - HarmBlockThreshold: {}, - createPartFromUri: vi.fn(), - } -}) - -const mocks = { - generateContent: vi.mocked(GoogleGenAI.prototype.models.generateContent), - listModels: vi.mocked(GoogleGenAI.prototype.models.list), - uploadFile: vi.mocked(GoogleGenAI.prototype.files.upload), - getFile: vi.mocked(GoogleGenAI.prototype.files.get), - fileTypeFromFile: vi.mocked(fileTypeFromFile), - fileTypeFromBuffer: vi.mocked(fileTypeFromBuffer), - createPartFromUri: vi.mocked(createPartFromUri), - readFile: vi.mocked(readFile), -} - -const spies: { [key: string]: MockInstance } = {} - -describe('lib/llm/google.ts', () => { - afterEach(() => { - Object.values(mocks).forEach((mock) => { - mock.mockReset() - }) - - Object.keys(spies).forEach((key) => { - spies[key].mockRestore() - delete spies[key] - }) - }) - - test('pieceFrontMatterFromPrompt', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const frontmatter = { title: 'value' } - const responseText = JSON.stringify(frontmatter) - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt) - - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt]), - config: expect.any(Object), - model: expect.any(String), - }) - expect(generatedFrontmatter).toEqual(frontmatter) - }) - - test('pieceFrontMatterFromPrompt strips null fields', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const frontmatter = { title: 'value', keywords: null } - const responseText = JSON.stringify(frontmatter) - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt) - - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt]), - config: expect.any(Object), - model: expect.any(String), - }) - expect(generatedFrontmatter).toEqual({ title: 'value' }) - }) - - test('pieceFrontMatterFromPrompt keeps schema-permitted empty strings', async () => { - mocks.generateContent.mockResolvedValueOnce({ - text: JSON.stringify({ title: '', keywords: '' }), - } as GenerateContentResponse) - - await expect(pieceFrontMatterFromPrompt('apiKey', makeSchema('books'), 'prompt')).resolves.toEqual({ - title: '', - keywords: '', - }) - }) - - test('pieceFrontMatterFromPrompt rejects a required null field', async () => { - mocks.generateContent.mockResolvedValueOnce({ - text: JSON.stringify({ title: null }), - } as GenerateContentResponse) - - await expect(pieceFrontMatterFromPrompt('apiKey', makeSchema('books'), 'prompt')).rejects.toThrow( - "must have required property 'title'" - ) - }) - - test('pieceFrontMatterFromPrompt rejects an empty result missing required fields', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const responseText = undefined - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - - await expect(pieceFrontMatterFromPrompt(apiKey, schema, prompt)).rejects.toThrow( - 'Generated frontmatter does not match schema' - ) - }) - - test('pieceFrontMatterFromPrompt rejects the wrong types', async () => { - mocks.generateContent.mockResolvedValueOnce({ - text: JSON.stringify({ title: 42 }), - } as GenerateContentResponse) - - await expect(pieceFrontMatterFromPrompt('apiKey', makeSchema('books'), 'prompt')).rejects.toThrow( - 'Generated frontmatter does not match schema: /title must be string' - ) - }) - - test('pieceFrontMatterFromPrompt rejects non-object output', async () => { - for (const text of ['null', '[]', '"text"']) { - mocks.generateContent.mockResolvedValueOnce({ text } as GenerateContentResponse) - await expect(pieceFrontMatterFromPrompt('apiKey', makeSchema('books'), 'prompt')).rejects.toThrow( - 'Generated frontmatter does not match schema: / must be object' - ) - } - }) - - test('pieceFrontMatterFromPrompt checks schema constraints', async () => { - const schema = makeSchema('books') - schema.properties.title = { type: 'string', minLength: 3 } as typeof schema.properties.title - mocks.generateContent.mockResolvedValueOnce({ - text: JSON.stringify({ title: 'a' }), - } as GenerateContentResponse) - - await expect(pieceFrontMatterFromPrompt('apiKey', schema, 'prompt')).rejects.toThrow( - 'Generated frontmatter does not match schema: /title must NOT have fewer than 3 characters' - ) - }) - - test('generatePieceFrontmatter with a binary file', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const file = '/path/to/file.pdf' - const uri = 'gs://another/path/to/file.pdf' - const name = 'file.pdf' - const mimeType = 'application/pdf' - const frontmatter = { title: 'value' } - const responseText = JSON.stringify(frontmatter) - const fileContent = 'fileContent' as Part - - vi.useFakeTimers() - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.uploadFile.mockResolvedValue({ name, uri, mimeType }) - mocks.getFile.mockResolvedValueOnce({ state: 'PROCESSING' as FileState }) - mocks.getFile.mockResolvedValueOnce({ uri, mimeType, state: 'ACTIVE' as FileState }) - mocks.fileTypeFromFile.mockResolvedValueOnce({ mime: mimeType, ext: 'pdf' }) - mocks.createPartFromUri.mockReturnValue(fileContent) - - vi.runAllTimersAsync() - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt, [file]) - - expect(mocks.uploadFile).toHaveBeenCalledWith({ file, config: { mimeType } }) - expect(mocks.getFile).toHaveBeenCalledWith({ name }) - expect(mocks.createPartFromUri).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt, fileContent]), - config: expect.any(Object), - model: expect.any(String), - }) - expect(generatedFrontmatter).toEqual(frontmatter) - - vi.useRealTimers() - }) - - test('generatePieceFrontmatter with a text file', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const file = '/path/to/file.html' - const frontmatter = { title: 'value' } - const responseText = JSON.stringify(frontmatter) - const fileContent = 'fileContent' - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.fileTypeFromFile.mockResolvedValueOnce(undefined) - mocks.readFile.mockResolvedValueOnce(fileContent) - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt, [file]) - - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt, expect.stringContaining(fileContent)]), - config: expect.any(Object), - model: expect.any(String), - }) - expect(generatedFrontmatter).toEqual(frontmatter) - - vi.useRealTimers() - }) - - test('generatePieceFrontmatter with a text Buffer', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const frontmatter = { title: 'value' } - const responseText = JSON.stringify(frontmatter) - const buffer = Buffer.from('buffer data') - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.fileTypeFromBuffer.mockResolvedValueOnce(undefined) - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt, [buffer]) - - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt, expect.stringContaining(buffer.toString())]), - config: expect.any(Object), - model: expect.any(String), - }) - - expect(generatedFrontmatter).toEqual(frontmatter) - }) - - test('generatePieceFrontmatter with a binary Buffer', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const name = 'file.pdf' - const uri = 'gs://another/path/to/file.pdf' - const mimeType = 'application/pdf' - const frontmatter = { title: 'value' } - const responseText = JSON.stringify(frontmatter) - const buffer = Buffer.from('buffer data') - const blob = new Blob([buffer]) - const fileContent = 'fileContent' as Part - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.uploadFile.mockResolvedValue({ name, uri, mimeType }) - mocks.getFile.mockResolvedValue({ uri, mimeType, state: 'ACTIVE' as FileState }) - mocks.fileTypeFromBuffer.mockResolvedValueOnce({ mime: mimeType, ext: 'pdf' }) - mocks.createPartFromUri.mockReturnValue(fileContent) - - const generatedFrontmatter = await pieceFrontMatterFromPrompt(apiKey, schema, prompt, [buffer]) - - expect(mocks.uploadFile).toHaveBeenCalledWith({ file: blob, config: { mimeType } }) - expect(mocks.getFile).toHaveBeenCalledWith({ name }) - expect(mocks.generateContent).toHaveBeenCalledTimes(1) - expect(mocks.generateContent).toHaveBeenCalledWith({ - contents: expect.arrayContaining([prompt, fileContent]), - config: expect.any(Object), - model: expect.any(String), - }) - - expect(generatedFrontmatter).toEqual(frontmatter) - }) - - test('generatePieceFrontmatter binary file fails to process upload', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const file = '/path/to/file.pdf' - const name = 'file.pdf' - const mimeType = 'application/pdf' - const frontmatter = { field: 'value' } - const responseText = JSON.stringify(frontmatter) - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.uploadFile.mockResolvedValue({ name, state: 'FAILED' as FileState }) - mocks.getFile.mockResolvedValue({}) - mocks.fileTypeFromFile.mockResolvedValueOnce({ mime: mimeType, ext: 'pdf' }) - - const generating = pieceFrontMatterFromPrompt(apiKey, schema, prompt, [file]) - - expect(generating).rejects.toThrowError() - }) - - test('generatePieceFrontmatter file fails to extract', async () => { - const apiKey = 'apiKey' - const schema = makeSchema('books') - const prompt = 'prompt' - const file = '/path/to/file.pdf' - const name = 'file.pdf' - const mimeType = 'application/pdf' - const frontmatter = { field: 'value' } - const responseText = JSON.stringify(frontmatter) - - mocks.generateContent.mockResolvedValueOnce({ - text: responseText, - } as GenerateContentResponse) - mocks.uploadFile.mockResolvedValue({ name, uri: undefined }) - mocks.getFile.mockResolvedValue({}) - mocks.fileTypeFromFile.mockResolvedValueOnce({ mime: mimeType, ext: 'pdf' }) - - const generating = pieceFrontMatterFromPrompt(apiKey, schema, prompt, [file]) - - expect(generating).rejects.toThrowError() - }) - - test('uses a five-minute timeout for Google requests', async () => { - vi.mocked(GoogleGenAI).mockClear() - mocks.listModels.mockResolvedValueOnce({} as never) - - await validateApiKey('apiKey') - - expect(GoogleGenAI).toHaveBeenCalledWith({ - apiKey: 'apiKey', - httpOptions: { timeout: 5 * 60 * 1000 }, - }) - }) - - test('validateApiKey resolves ok when the key can list models', async () => { - mocks.listModels.mockResolvedValueOnce({} as never) - - const result = await validateApiKey('apiKey') - - expect(result).toEqual({ ok: true }) - }) - - test('validateApiKey resolves not-ok when the request fails', async () => { - mocks.listModels.mockRejectedValueOnce(new Error('API key not valid')) - - const result = await validateApiKey('apiKey') - - expect(result).toEqual({ ok: false, reason: 'API key not valid' }) - }) -}) diff --git a/apps/core/src/llm/google.ts b/apps/core/src/llm/google.ts deleted file mode 100644 index 4dfe4a99..00000000 --- a/apps/core/src/llm/google.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { fileTypeFromBuffer } from 'file-type' -import { fileTypeFromFile } from 'file-type/node' -import type { - ContentListUnion, - SafetySetting} from '@google/genai'; -import { - createPartFromUri, - GoogleGenAI, - HarmCategory, - HarmBlockThreshold -} from '@google/genai' -import { readFile } from 'fs/promises' -import path from 'path' -import type { JSONSchemaType } from 'ajv' -import compile from '../lib/ajv.js' -import { type PieceFrontmatter } from '../pieces/index.js' - -const MODEL_NAME = 'gemini-2.5-flash' -const REQUEST_TIMEOUT_MS = 5 * 60 * 1000 - -const safetySettings: SafetySetting[] = [ - { - category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - { - category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, - { - category: HarmCategory.HARM_CATEGORY_CIVIC_INTEGRITY, - threshold: HarmBlockThreshold.BLOCK_NONE, - }, -] - -function getClient(apiKey: string) { - return new GoogleGenAI({ apiKey, httpOptions: { timeout: REQUEST_TIMEOUT_MS } }) -} - -async function extractPartFromFile(file: string | Buffer, genAI: GoogleGenAI) { - const isBuffer = Buffer.isBuffer(file) - const fileType = isBuffer ? await fileTypeFromBuffer(file) : await fileTypeFromFile(file) - const mimeType = fileType?.mime - - // assume it's non binary - if (!mimeType) { - const fileContent = isBuffer ? file.toString() : await readFile(file, 'utf-8') - const fileName = isBuffer ? 'string buffer of downloaded file' : path.basename(file) - const data = [ - `---[start] embedding ${fileName}---`, - fileContent, - `---[end] embedding of ${fileName}---`, - ] - - return data.join('\n') - } else { - const uploadedFile = await genAI.files.upload({ - file: isBuffer ? new Blob([file]) : file, - config: { - mimeType, - }, - }) - - // Wait for the file to be processed. - let getFile = await genAI.files.get({ name: uploadedFile.name as string }) - - while (getFile.state === 'PROCESSING') { - getFile = await genAI.files.get({ name: uploadedFile.name as string }) - - await new Promise((resolve) => { - setTimeout(resolve, 5000) - }) - } - - if (uploadedFile.state === 'FAILED') { - throw new Error('File processing failed.') - } - - if (uploadedFile.uri && uploadedFile.mimeType) { - const fileContent = createPartFromUri(uploadedFile.uri, uploadedFile.mimeType) - return fileContent - } - } - - throw new Error('File processing failed.') -} - -function omitNullFrontmatterFields(value: unknown): unknown { - if (value === null) return value - if (typeof value !== 'object') return value - if (Array.isArray(value)) return value - - return Object.fromEntries( - Object.entries(value).filter(([, field]) => field !== null) - ) -} - -function parseAndValidateFrontmatter( - metadata: string, - schema: JSONSchemaType -): PieceFrontmatter { - const parsed: unknown = JSON.parse(metadata) - const frontmatter = omitNullFrontmatterFields(parsed) - const validate = compile(schema) - if (!validate(frontmatter)) { - const error = validate.errors?.[0] - throw new Error( - `Generated frontmatter does not match schema: ${error?.instancePath || '/'} ${error?.message}` - ) - } - - return frontmatter -} - -async function pieceFrontMatterFromPrompt( - apiKey: string, - schema: JSONSchemaType, - prompt: string, - files?: Array -) { - const genAI = getClient(apiKey) - - const content: ContentListUnion = [prompt] - - if (files) { - for (const file of files) { - const part = await extractPartFromFile(file, genAI) - content.push(part) - } - } - - const result = await genAI.models.generateContent({ - model: MODEL_NAME, - contents: content, - config: { - responseMimeType: 'application/json', - safetySettings: safetySettings, - responseJsonSchema: schema, - systemInstruction: `you are an assistant that helps generate JSON metadata for a record that will be added to a collection of similar records. - -if you are provided pdf attachments, images or other text based files, please prioritize them as inputs for generating metadata for the record. - -you are also given a responseJsonSchema to guide your output. each field in the schema has a description and examples to help guide what the intention of each field is and what values to expect.`, - }, - }) - - return parseAndValidateFrontmatter(result.text || '{}', schema) -} - -async function validateApiKey( - apiKey: string -): Promise<{ ok: true } | { ok: false; reason: string }> { - try { - const genAI = getClient(apiKey) - await genAI.models.list() - return { ok: true } - } catch (err) { - return { ok: false, reason: err instanceof Error ? err.message : String(err) } - } -} - -export { pieceFrontMatterFromPrompt, validateApiKey } diff --git a/apps/core/src/llm/google/client.ts b/apps/core/src/llm/google/client.ts new file mode 100644 index 00000000..f5b10385 --- /dev/null +++ b/apps/core/src/llm/google/client.ts @@ -0,0 +1,285 @@ +import { createReadStream } from 'node:fs' +import path from 'node:path' +import { fileTypeFromBuffer } from 'file-type' +import { + ApiError, + createPartFromUri, + GoogleGenAI, + HarmCategory, + HarmBlockThreshold, +} from '@google/genai' +import type { File as GeminiFile, GenerateContentResponse, Part } from '@google/genai' +import type { PieceFrontmatter, PieceFrontmatterSchema } from '../../pieces/utils/frontmatter.js' + +const MODEL_NAME = 'gemini-3.8-flash' +const REQUEST_TIMEOUT_MS = 5 * 60 * 1000 +const FILE_POLL_MS = 5000 +const CLEANUP_TIMEOUT_MS = 10_000 + +export type GenerationProgress = { + phase: 'preparation' | 'generation' | 'validation' + message: string +} + +export type GenerationLimits = { + maxFiles: number + maxFileBytes: number + maxTotalFileBytes: number + /** Maximum accepted result text; checked after the SDK receives the response. */ + maxOutputBytes: number + /** Stop starting new status checks after this wait; active SDK requests may finish. */ + maxFileProcessingMs: number +} + +/** Core safety budgets, not measured web deployment limits. */ +export const DEFAULT_GENERATION_LIMITS: Readonly = { + maxFiles: 10, + maxFileBytes: 50_000_000, + maxTotalFileBytes: 100_000_000, + maxOutputBytes: 1_000_000, + maxFileProcessingMs: 5 * 60 * 1000, +} + +export type GenerationOptions = { + files?: Array + limits?: Partial + onProgress?: (progress: GenerationProgress) => void | Promise + onWarning?: (message: string) => void | Promise +} + +type GenerationTask = { + prompt: string[] + systemInstruction: string + schema?: PieceFrontmatterSchema + decode: (text: string) => T +} + +type GenerationContext = { + client: GoogleGenAI + limits: GenerationLimits + options: GenerationOptions + uploaded: Set +} + +function getClient(apiKey: string) { + return new GoogleGenAI({ apiKey, httpOptions: { timeout: REQUEST_TIMEOUT_MS } }) +} + +export async function validateApiKey( + apiKey: string +): Promise<{ ok: true } | { ok: false; reason: string }> { + try { + await getClient(apiKey).models.list() + return { ok: true } + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) } + } +} + +function generationLimits(options: GenerationOptions, fileCount: number): GenerationLimits { + const limits = { ...DEFAULT_GENERATION_LIMITS, ...options.limits } + for (const [key, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value <= 0) + throw new Error(`Invalid generation limit: ${key}.`) + } + if (fileCount > limits.maxFiles) throw new Error('Too many generation attachments.') + return limits +} + +function progress(context: GenerationContext, phase: GenerationProgress['phase'], message: string) { + // Losing a progress subscriber must not interrupt accepted work. + try { + void Promise.resolve(context.options.onProgress?.({ phase, message })).catch(() => {}) + } catch { + /* best-effort notification */ + } +} + +async function readInput(file: string | Buffer, maxBytes: number): Promise> { + if (Buffer.isBuffer(file)) { + if (file.length > maxBytes) throw new Error('Attachment exceeds the file or total byte limit.') + return Buffer.from(file) + } + const chunks: Buffer[] = [] + let bytes = 0 + for await (const chunk of createReadStream(file, { highWaterMark: 64 * 1024 })) { + bytes += chunk.length + if (bytes > maxBytes) throw new Error('Attachment exceeds the file or total byte limit.') + chunks.push(chunk) + } + return Buffer.concat(chunks, bytes) +} + +async function uploadFile( + client: GoogleGenAI, + bytes: Uint8Array, + mimeType: string +): Promise { + let uploaded: GeminiFile + try { + uploaded = await client.files.upload({ + file: new Blob([bytes], { type: mimeType }), + config: { mimeType, displayName: 'luzzle-generation-input' }, + }) + } catch (error) { + const status = error instanceof ApiError ? ` (HTTP ${error.status})` : '' + throw new Error(`Gemini file upload failed${status}.`) + } + if (typeof uploaded?.name !== 'string' || !uploaded.name) { + throw new Error('Gemini file upload returned no file name.') + } + return uploaded.name +} + +async function waitForFile(name: string, context: GenerationContext): Promise { + const stopPollingAt = Date.now() + context.limits.maxFileProcessingMs + while (Date.now() < stopPollingAt) { + const file = await context.client.files.get({ name }) + if (file.state === 'ACTIVE' && file.uri && file.mimeType) + return createPartFromUri(file.uri, file.mimeType) + if (file.state !== 'PROCESSING') + throw new Error('Attachment processing failed or returned incomplete file metadata.') + const remaining = stopPollingAt - Date.now() + if (remaining <= 0) break + await new Promise((resolve) => setTimeout(resolve, Math.min(FILE_POLL_MS, remaining))) + } + throw new Error('Attachment processing exceeded the wait limit.') +} + +async function prepareAttachments(context: GenerationContext, files: Array) { + const { limits } = context + const parts: Array = [] + let totalBytes = 0 + for (const [index, file] of files.entries()) { + progress(context, 'preparation', `Preparing attachment ${index + 1} of ${files.length}.`) + const bytes = await readInput( + file, + Math.min(limits.maxFileBytes, limits.maxTotalFileBytes - totalBytes) + ) + totalBytes += bytes.length + const type = await fileTypeFromBuffer(bytes) + if (!type) { + const name = typeof file === 'string' ? path.basename(file) : 'text attachment' + parts.push( + `---[start] embedding ${name}---\n${bytes.toString('utf8')}\n---[end] embedding ${name}---` + ) + continue + } + const name = await uploadFile(context.client, bytes, type.mime) + context.uploaded.add(name) + progress(context, 'preparation', `Processing attachment ${index + 1} of ${files.length}.`) + parts.push(await waitForFile(name, context)) + } + return parts +} + +function warnAboutCleanup(options: GenerationOptions) { + const message = 'Could not delete a temporary generation file; provider expiry still applies.' + const warn = options.onWarning ?? console.warn + try { + void Promise.resolve(warn(message)).catch(() => console.warn(message)) + } catch { + console.warn(message) + } +} + +async function cleanupFiles(context: GenerationContext) { + await Promise.all( + [...context.uploaded].map(async (name) => { + try { + await context.client.files.delete({ + name, + config: { httpOptions: { timeout: CLEANUP_TIMEOUT_MS } }, + }) + } catch { + // Provider errors may include credentials or user content; don't expose them. + warnAboutCleanup(context.options) + } + }) + ) +} + +function completedText(response: GenerateContentResponse, maxOutputBytes: number): string { + if ( + response.promptFeedback?.blockReason && + response.promptFeedback.blockReason !== 'BLOCKED_REASON_UNSPECIFIED' + ) { + throw new Error('Generation was blocked by the provider.') + } + const candidates = response.candidates ?? [] + if (candidates.length !== 1 || (candidates[0].index !== undefined && candidates[0].index !== 0)) { + throw new Error('Generation returned unexpected candidates.') + } + const candidate = candidates[0] + if (candidate.safetyRatings?.some((rating) => rating.blocked)) + throw new Error('Generation was blocked by the provider.') + if (candidate.finishReason !== 'STOP') { + throw new Error( + `Generation did not finish successfully (${candidate.finishReason ?? 'missing finish reason'}).` + ) + } + let text = '' + for (const part of candidate.content?.parts ?? []) { + if (part.thought) continue + if (typeof part.text !== 'string') throw new Error('Generation returned non-text output.') + text += part.text + } + if (!text.trim()) throw new Error('Generation returned empty output.') + if (Buffer.byteLength(text, 'utf8') > maxOutputBytes) + throw new Error('Generated output exceeds the byte limit.') + return text +} + +async function generateText( + context: GenerationContext, + task: GenerationTask, + attachments: Array +) { + progress(context, 'generation', 'Generating content.') + const response = await context.client.models.generateContent({ + model: MODEL_NAME, + contents: [...task.prompt, ...attachments], + config: { + candidateCount: 1, + systemInstruction: task.systemInstruction, + ...(task.schema + ? { responseMimeType: 'application/json', responseJsonSchema: task.schema } + : {}), + safetySettings: [ + { + category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + { + category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, + threshold: HarmBlockThreshold.BLOCK_NONE, + }, + ], + }, + }) + return completedText(response, context.limits.maxOutputBytes) +} + +export async function runGeneration( + apiKey: string, + task: GenerationTask, + options: GenerationOptions +): Promise { + const files = [...(options.files ?? [])] + const limits = generationLimits(options, files.length) + const context: GenerationContext = { + client: getClient(apiKey), + limits, + options, + uploaded: new Set(), + } + try { + progress(context, 'preparation', 'Preparing generation inputs.') + const attachments = await prepareAttachments(context, files) + const text = await generateText(context, task, attachments) + progress(context, 'validation', 'Validating the completed result.') + return task.decode(text) + } finally { + await cleanupFiles(context) + } +} diff --git a/apps/core/src/llm/google/generation.test.ts b/apps/core/src/llm/google/generation.test.ts new file mode 100644 index 00000000..58838075 --- /dev/null +++ b/apps/core/src/llm/google/generation.test.ts @@ -0,0 +1,801 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest' +import { Readable } from 'node:stream' +import type { ReadStream, createReadStream } from 'node:fs' +import type { fileTypeFromBuffer } from 'file-type' +import type * as GeminiSDK from '@google/genai' +import type { JSONSchemaType } from 'ajv' +import { + ApiError, + FileState, + FinishReason, + GenerateContentResponse, + GoogleGenAI, +} from '@google/genai' +import type { PieceFrontmatter } from '../../pieces/utils/frontmatter.js' +import { + DEFAULT_GENERATION_LIMITS, + pieceFrontMatterFromPrompt, + generatePieceMetadata, + generateFieldValue, + generateBody, + validateApiKey, +} from './index.js' +import type { GenerationOptions } from './index.js' + +const mocks = vi.hoisted(() => ({ + generate: vi.fn(), + get: vi.fn(), + delete: vi.fn(), + list: vi.fn(), + upload: vi.fn(), + fileType: vi.fn(), + readStream: vi.fn(), +})) + +vi.mock('@google/genai', async (original) => ({ + ...(await original()), + GoogleGenAI: vi.fn(function () { + return { + models: { generateContent: mocks.generate, list: mocks.list }, + files: { upload: mocks.upload, get: mocks.get, delete: mocks.delete }, + } + }), +})) +vi.mock('file-type', () => ({ fileTypeFromBuffer: mocks.fileType })) +vi.mock('node:fs', () => ({ createReadStream: mocks.readStream })) + +function schema(): JSONSchemaType { + return { + title: 'books', + type: 'object', + properties: { title: { type: 'string' }, keywords: { type: 'string', nullable: true } }, + required: ['title'], + additionalProperties: false, + } as unknown as JSONSchemaType +} +function response(text?: string, finishReason = FinishReason.STOP): GenerateContentResponse { + return Object.assign(new GenerateContentResponse(), { + candidates: [{ content: { parts: text === undefined ? [] : [{ text }] }, finishReason }], + }) +} +function respond(text: string, finishReason = FinishReason.STOP) { + mocks.generate.mockResolvedValue(response(text, finishReason)) +} +function binary() { + mocks.fileType.mockResolvedValue({ mime: 'application/pdf', ext: 'pdf' }) +} +function deferred() { + let resolve!: (value: T) => void + let reject!: (error: unknown) => void + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} +async function until(condition: () => boolean) { + for (let i = 0; i < 100; i++) { + if (condition()) return + await Promise.resolve() + } + throw new Error('Expected asynchronous operation did not start') +} +const run = (options: GenerationOptions = {}) => + generatePieceMetadata('test-key', schema(), 'My instructions', options) + +beforeEach(() => { + Object.values(mocks).forEach((mock) => mock.mockReset()) + vi.mocked(GoogleGenAI).mockClear() + respond('{"title":"generated"}') + mocks.fileType.mockResolvedValue(undefined) + mocks.upload.mockResolvedValue({ name: 'files/uploaded' }) + mocks.get.mockResolvedValue({ + state: FileState.ACTIVE, + uri: 'https://provider/fresh-uri', + mimeType: 'application/pdf', + }) + mocks.delete.mockResolvedValue({}) + mocks.list.mockResolvedValue({} as never) +}) +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() +}) + +describe('validated generation', () => { + test('retains the original metadata instruction and adds only the field restriction', async () => { + const original = [ + 'you are an assistant that helps generate JSON metadata for a record that will be added to a collection of similar records.', + 'if you are provided pdf attachments, images or other text based files, please prioritize them as inputs for generating metadata for the record.', + 'you are also given a responseJsonSchema to guide your output. each field in the schema has a description and examples to help guide what the intention of each field is and what values to expect.', + ].join('\n\n') + await run() + expect(mocks.generate.mock.calls[0][0].config!.systemInstruction).toBe(original) + + respond('{"keywords":"new"}') + await generateFieldValue('key', { schema: schema(), key: 'keywords', source: '' }) + expect(mocks.generate.mock.calls[1][0].config!.systemInstruction).toBe( + `${original}\n\ngenerate only the "keywords" field.` + ) + }) + + test('reports semantic phases and returns validated metadata', async () => { + const onProgress = vi.fn() + await expect(run({ onProgress })).resolves.toEqual({ title: 'generated' }) + expect(onProgress.mock.calls.map(([event]) => event.phase)).toEqual([ + 'preparation', + 'generation', + 'validation', + ]) + expect(mocks.generate).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + model: 'gemini-3.8-flash', + contents: ['My instructions'], + config: expect.objectContaining({ + candidateCount: 1, + responseMimeType: 'application/json', + responseJsonSchema: schema(), + }), + }) + ) + }) + + test.each([ + { text: '{"title":"new"}', valid: true }, + { text: '{"title":42}', valid: false }, + ])( + 'publishes no result until the full response and validation complete: $text', + async ({ text, valid }) => { + const pending = deferred() + mocks.generate.mockReturnValue(pending.promise) + const onProgress = vi.fn() + const resolved = vi.fn() + const rejected = vi.fn() + const generating = run({ onProgress }).then(resolved, rejected) + await until(() => mocks.generate.mock.calls.length === 1) + expect(resolved).not.toHaveBeenCalled() + expect(rejected).not.toHaveBeenCalled() + expect(onProgress.mock.calls.map(([event]) => event.phase)).toEqual([ + 'preparation', + 'generation', + ]) + pending.resolve(response(text)) + await generating + if (!valid) { + expect(resolved).not.toHaveBeenCalled() + expect(rejected).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ message: expect.stringContaining('does not match schema') }) + ) + } else { + expect(resolved).toHaveBeenCalledExactlyOnceWith({ title: 'new' }) + expect(rejected).not.toHaveBeenCalled() + } + } + ) + + test('compatibility wrapper keeps full-metadata null stripping and empty strings', async () => { + respond('{"title":"","keywords":null}') + await expect(pieceFrontMatterFromPrompt('test-key', schema(), 'prompt')).resolves.toEqual({ + title: '', + }) + respond('{"title":"value","keywords":""}') + await expect(pieceFrontMatterFromPrompt('test-key', schema(), 'prompt')).resolves.toEqual({ + title: 'value', + keywords: '', + }) + }) + + test.each([ + ['{"title":null}', "must have required property 'title'"], + ['{"title":42}', '/title must be string'], + ['{"title":"new","extra":true}', 'must NOT have additional properties'], + ['null', 'must be object'], + ['[]', 'must be object'], + ['"text"', 'must be object'], + ['{"title":', 'not valid JSON'], + ])('rejects invalid completed metadata %s', async (text, error) => { + respond(text) + await expect(pieceFrontMatterFromPrompt('test-key', schema(), 'prompt')).rejects.toThrow(error) + }) + + test('validates schema constraints and core formats', async () => { + const metadataSchema = schema() + metadataSchema.properties.title = { type: 'string', format: 'date' } + respond('{"title":"not-a-date"}') + await expect(generatePieceMetadata('key', metadataSchema, 'prompt')).rejects.toThrow( + 'must match format' + ) + }) + + test('generates one field without imposing other required fields or stripping its null value', async () => { + respond('{"keywords":null}') + const source = '---\ntitle: unsaved title\nkeywords: old\n---\nunsaved notes' + const result = await generateFieldValue('key', { + schema: schema(), + key: 'keywords', + source, + instructions: 'Choose keywords', + }) + expect(result).toBeNull() + expect(mocks.generate.mock.calls[0][0]).toMatchObject({ + contents: ['Choose keywords', expect.stringContaining(JSON.stringify(source))], + config: { + responseJsonSchema: { + type: 'object', + properties: { keywords: { type: 'string', nullable: true } }, + required: ['keywords'], + additionalProperties: false, + }, + }, + }) + }) + + test.each(['{}', '{"keywords":42}', '{"keywords":"value","title":"extra"}'])( + 'rejects invalid field output %s', + async (text) => { + respond(text) + await expect( + generateFieldValue('key', { + schema: schema(), + key: 'keywords', + source: '', + }) + ).rejects.toThrow('Generated field does not match schema') + } + ) + + test('requires an own field value, even for names inherited from Object.prototype', async () => { + const fieldSchema = { + type: 'object', + properties: { constructor: true }, + } as unknown as JSONSchemaType + respond('{}') + await expect( + generateFieldValue('key', { + schema: fieldSchema, + key: 'constructor', + source: '', + }) + ).rejects.toThrow('missing the requested property') + }) + + test('both operations snapshot their schema and context before invoking callbacks', async () => { + const metadataSchema = schema() + const metadata = await generatePieceMetadata('key', metadataSchema, '', { + onProgress: () => { + metadataSchema.properties.title = { type: 'number' } + }, + }) + expect(metadata).toEqual({ title: 'generated' }) + + const fieldRequest = { + schema: schema(), + key: 'keywords', + source: 'original context', + } + respond('{"keywords":"new"}') + const value = await generateFieldValue('key', fieldRequest, { + onProgress: () => { + fieldRequest.key = 'title' + fieldRequest.source = 'changed context' + }, + }) + expect(value).toBe('new') + expect(mocks.generate.mock.calls[1][0].contents).toContainEqual( + expect.stringContaining('original context') + ) + }) + + test('supports a complete array or object value', async () => { + const arraySchema = schema() + arraySchema.properties.title = { + type: 'array', + items: { type: 'object', properties: { name: { type: 'string' } }, required: ['name'] }, + } as typeof arraySchema.properties.title + respond('{"title":[{"name":"new"}]}') + await expect( + generateFieldValue('key', { schema: arraySchema, key: 'title', source: '' }) + ).resolves.toEqual([{ name: 'new' }]) + }) + + test('body generation returns Markdown exactly and does not request metadata JSON', async () => { + respond('\n# Notes\n\n trailing whitespace \n') + const source = 'unsaved source' + await expect(generateBody('key', { source })).resolves.toBe( + '\n# Notes\n\n trailing whitespace \n' + ) + const config = mocks.generate.mock.calls[0][0].config! + expect(config.responseJsonSchema).toBeUndefined() + expect(config.responseMimeType).toBeUndefined() + expect(config.systemInstruction).toContain('Markdown') + expect(mocks.generate.mock.calls[0][0].contents).toContainEqual( + expect.stringContaining(JSON.stringify(source)) + ) + }) + + test.each([undefined, '', ' \n '])('rejects empty completed body output %j', async (text) => { + mocks.generate.mockResolvedValue(response(text)) + await expect(generateBody('key', { source: '' })).rejects.toThrow('empty output') + }) + + test.each([FinishReason.MAX_TOKENS, FinishReason.SAFETY, FinishReason.OTHER])( + 'rejects provider finish reason %s even with valid JSON', + async (reason) => { + respond('{"title":"new"}', reason) + await expect(run()).rejects.toThrow('Generation did not finish successfully') + } + ) + + test('rejects a missing finish reason even with valid JSON', async () => { + const result = response('{"title":"new"}') + delete result.candidates![0].finishReason + mocks.generate.mockResolvedValue(result) + await expect(run()).rejects.toThrow('Generation did not finish successfully') + }) + + test('rejects a completed response without content instead of inventing empty metadata', async () => { + mocks.generate.mockResolvedValue( + Object.assign(new GenerateContentResponse(), { + candidates: [{ finishReason: FinishReason.STOP }], + }) + ) + await expect(run()).rejects.toThrow('empty output') + }) + + test('ignores hidden thought text and combines visible text parts', async () => { + const result = response() + result.candidates![0].content!.parts = [ + { text: 'private reasoning', thought: true }, + { text: '{"title":' }, + { text: '"new"}' }, + ] + mocks.generate.mockResolvedValue(result) + await expect(run()).resolves.toEqual({ title: 'new' }) + }) + + test.each([{}, { candidates: [] }, { candidates: [{ index: 1 }] }, { candidates: [{}, {}] }])( + 'rejects missing or unexpected candidates: %j', + async (data) => { + mocks.generate.mockResolvedValue(Object.assign(new GenerateContentResponse(), data)) + await expect(run()).rejects.toThrow('Generation returned unexpected candidates.') + } + ) + + test.each([ + { promptFeedback: { blockReason: 'SAFETY' } }, + { candidates: [{ safetyRatings: [{ blocked: true }], finishReason: FinishReason.STOP }] }, + ])('rejects provider-blocked output: %j', async (data) => { + mocks.generate.mockResolvedValue(Object.assign(response('{"title":"new"}'), data)) + await expect(run()).rejects.toThrow('blocked') + }) + + test('rejects non-text output even alongside valid JSON', async () => { + const result = response('{"title":"new"}') + result.candidates![0].content!.parts!.push({ functionCall: { name: 'unexpected' } }) + mocks.generate.mockResolvedValue(result) + await expect(run()).rejects.toThrow('non-text') + }) + + test('allows an unspecified prompt block reason', async () => { + const result = Object.assign(response('{"title":"new"}'), { + promptFeedback: { blockReason: 'BLOCKED_REASON_UNSPECIFIED' }, + }) + mocks.generate.mockResolvedValue(result as GenerateContentResponse) + await expect(run()).resolves.toEqual({ title: 'new' }) + }) +}) + +describe('input preparation and cleanup', () => { + test('embeds text buffers without uploading or logging their contents', async () => { + const onProgress = vi.fn() + await run({ files: [Buffer.from('private text')], onProgress }) + expect(mocks.generate.mock.calls[0][0].contents).toContainEqual( + expect.stringContaining('private text') + ) + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.delete).not.toHaveBeenCalled() + expect(JSON.stringify(onProgress.mock.calls)).not.toContain('private text') + }) + + test('reads file paths through a bounded stream', async () => { + mocks.readStream.mockReturnValue( + Readable.from([Buffer.from('one'), Buffer.from('two')]) as ReadStream + ) + await pieceFrontMatterFromPrompt('key', schema(), 'prompt', ['/private/path/notes.txt']) + expect(mocks.readStream).toHaveBeenCalledWith('/private/path/notes.txt', { + highWaterMark: 65536, + }) + expect(mocks.generate.mock.calls[0][0].contents).toContainEqual( + expect.stringContaining('embedding notes.txt---\nonetwo') + ) + }) + + test('uses the most recently fetched ACTIVE state, URI and MIME type', async () => { + vi.useFakeTimers() + binary() + mocks.get.mockResolvedValueOnce({ + state: FileState.PROCESSING, + uri: 'stale', + mimeType: 'stale/type', + }) + mocks.get.mockResolvedValueOnce({ + state: FileState.ACTIVE, + uri: 'https://provider/latest', + mimeType: 'image/png', + }) + const generating = run({ files: [Buffer.from('binary')] }) + await until(() => mocks.get.mock.calls.length === 1) + await vi.advanceTimersByTimeAsync(4999) + expect(mocks.get).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + await expect(generating).resolves.toEqual({ title: 'generated' }) + expect(mocks.generate.mock.calls[0][0].contents).toContainEqual({ + fileData: { fileUri: 'https://provider/latest', mimeType: 'image/png' }, + }) + expect(GoogleGenAI).toHaveBeenCalledExactlyOnceWith({ + apiKey: 'test-key', + httpOptions: { timeout: 300000 }, + }) + expect(mocks.upload).toHaveBeenCalledExactlyOnceWith({ + file: expect.any(Blob), + config: { mimeType: 'application/pdf', displayName: 'luzzle-generation-input' }, + }) + const uploaded = mocks.upload.mock.calls[0][0].file as Blob + expect(await uploaded.text()).toBe('binary') + expect(mocks.get.mock.calls).toEqual([ + [{ name: 'files/uploaded' }], + [{ name: 'files/uploaded' }], + ]) + expect(mocks.delete).toHaveBeenCalledExactlyOnceWith({ + name: 'files/uploaded', + config: { httpOptions: { timeout: 10000 } }, + }) + expect(vi.getTimerCount()).toBe(0) + }) + + test.each([ + { state: FileState.FAILED }, + {}, + { state: FileState.ACTIVE, uri: 'missing-mime' }, + { state: FileState.ACTIVE, mimeType: 'missing-uri' }, + ])('rejects failed or incomplete processing and deletes the uploaded file: %j', async (file) => { + binary() + mocks.get.mockResolvedValue(file) + await expect(run({ files: [Buffer.from('binary')] })).rejects.toThrow( + 'Attachment processing failed' + ) + expect(mocks.generate).not.toHaveBeenCalled() + expect(mocks.delete).toHaveBeenCalledTimes(1) + }) + + test.each([undefined, null, '', 42])( + 'rejects an invalid uploaded file name: %j', + async (name) => { + binary() + mocks.upload.mockResolvedValue({ name } as Awaited>) + await expect(run({ files: [Buffer.from('binary')] })).rejects.toThrow( + 'Gemini file upload returned no file name.' + ) + expect(mocks.get).not.toHaveBeenCalled() + expect(mocks.generate).not.toHaveBeenCalled() + expect(mocks.delete).not.toHaveBeenCalled() + } + ) + + test.each([ + [new Error('private-key session-token private content'), 'Gemini file upload failed.'], + [ + new ApiError({ status: 403, message: 'private-key session-token private content' }), + 'Gemini file upload failed (HTTP 403).', + ], + ])('sanitizes SDK upload failures: %s', async (error, message) => { + binary() + mocks.upload.mockRejectedValue(error) + await expect(run({ files: [Buffer.from('binary')] })).rejects.toMatchObject({ message }) + expect(mocks.get).not.toHaveBeenCalled() + expect(mocks.generate).not.toHaveBeenCalled() + expect(mocks.delete).not.toHaveBeenCalled() + }) + + test('cleans known files when the SDK file lookup fails', async () => { + binary() + mocks.get.mockRejectedValue(new Error('lookup failed')) + await expect(run({ files: [Buffer.from('binary')] })).rejects.toThrow('lookup failed') + expect(mocks.delete).toHaveBeenCalledExactlyOnceWith({ + name: 'files/uploaded', + config: { httpOptions: { timeout: 10000 } }, + }) + expect(mocks.generate).not.toHaveBeenCalled() + }) + + test('cleans earlier files when a later upload fails', async () => { + binary() + mocks.upload + .mockResolvedValueOnce({ name: 'files/first' }) + .mockRejectedValueOnce(new Error('upload failed')) + await expect(run({ files: [Buffer.from('one'), Buffer.from('two')] })).rejects.toThrow( + 'Gemini file upload failed.' + ) + expect(mocks.delete).toHaveBeenCalledExactlyOnceWith({ + name: 'files/first', + config: { httpOptions: { timeout: 10000 } }, + }) + expect(mocks.generate).not.toHaveBeenCalled() + }) + + test('cleans every known file on invalid completion', async () => { + binary() + mocks.upload + .mockResolvedValueOnce({ name: 'files/first' }) + .mockResolvedValueOnce({ name: 'files/second' }) + respond('{"title":42}') + await expect(run({ files: [Buffer.from('one'), Buffer.from('two')] })).rejects.toThrow( + 'does not match schema' + ) + expect(mocks.delete.mock.calls.map(([arg]) => arg.name)).toEqual([ + 'files/first', + 'files/second', + ]) + }) + + test('cleans files if model startup fails', async () => { + binary() + mocks.generate.mockRejectedValue(new Error('provider unavailable')) + await expect(run({ files: [Buffer.from('one')] })).rejects.toThrow('provider unavailable') + expect(mocks.delete).toHaveBeenCalledTimes(1) + }) + + test('reports cleanup failures without exposing provider errors or changing the valid result', async () => { + binary() + mocks.delete.mockRejectedValue(new Error('API key secret; private prompt')) + const onWarning = vi.fn() + await expect(run({ files: [Buffer.from('one')], onWarning })).resolves.toEqual({ + title: 'generated', + }) + expect(onWarning).toHaveBeenCalledExactlyOnceWith( + 'Could not delete a temporary generation file; provider expiry still applies.' + ) + }) + + test('uses safe console warnings by default and if the warning callback throws', async () => { + binary() + mocks.delete.mockRejectedValue(new Error('secret')) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + await run({ files: [Buffer.from('one')] }) + await run({ + files: [Buffer.from('two')], + onWarning: () => { + throw new Error('observer failed') + }, + }) + expect(warn).toHaveBeenCalledTimes(2) + expect(JSON.stringify(warn.mock.calls)).not.toContain('secret') + }) + test('falls back to a safe warning when an asynchronous warning callback rejects', async () => { + binary() + mocks.delete.mockRejectedValue(new Error('private provider error')) + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}) + const onWarning = vi.fn().mockRejectedValue(new Error('private observer error')) + await expect(run({ files: [Buffer.from('one')], onWarning })).resolves.toEqual({ + title: 'generated', + }) + expect(onWarning).toHaveBeenCalledOnce() + expect(warn).toHaveBeenCalledExactlyOnceWith( + 'Could not delete a temporary generation file; provider expiry still applies.' + ) + }) + + test('does not wait for warning callbacks or skip cleanup of other known files', async () => { + binary() + mocks.upload + .mockResolvedValueOnce({ name: 'files/first' }) + .mockResolvedValueOnce({ name: 'files/second' }) + mocks.delete.mockRejectedValueOnce(new Error('cleanup failed')) + const onWarning = vi.fn(() => new Promise(() => {})) + await expect( + run({ files: [Buffer.from('one'), Buffer.from('two')], onWarning }) + ).resolves.toEqual({ title: 'generated' }) + expect(mocks.delete.mock.calls.map(([arg]) => arg.name)).toEqual([ + 'files/first', + 'files/second', + ]) + expect(onWarning).toHaveBeenCalledOnce() + }) +}) + +describe('input and output bounds', () => { + test.each([0, -1, NaN, Infinity, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects invalid limits before provider work: %s', + async (maxFiles) => { + await expect(run({ limits: { maxFiles } })).rejects.toThrow('Invalid generation limit') + await expect(run({ limits: { maxFileProcessingMs: maxFiles } })).rejects.toThrow( + 'Invalid generation limit' + ) + expect(GoogleGenAI).not.toHaveBeenCalled() + } + ) + + test('rejects excess file counts and retains the default processing wait limit', async () => { + await expect( + run({ files: [Buffer.from('1'), Buffer.from('2')], limits: { maxFiles: 1 } }) + ).rejects.toThrow('Too many') + expect(GoogleGenAI).not.toHaveBeenCalled() + expect(DEFAULT_GENERATION_LIMITS.maxFileProcessingMs).toBe(300000) + }) + + test('rejects invalid schema or unknown field before provider work', async () => { + await expect( + generateFieldValue('key', { schema: schema(), key: 'missing', source: '' }) + ).rejects.toThrow('top-level property') + const invalid = schema() + invalid.properties.title = { type: 'string', format: 'unknown' } + await expect(generatePieceMetadata('key', invalid, '')).rejects.toThrow('unknown format') + expect(GoogleGenAI).not.toHaveBeenCalled() + }) + + test('bounds buffers and combined input bytes before starting further uploads', async () => { + await expect(run({ files: [Buffer.from('123')], limits: { maxFileBytes: 2 } })).rejects.toThrow( + 'byte limit' + ) + await expect( + run({ files: [Buffer.from('12'), Buffer.from('34')], limits: { maxTotalFileBytes: 3 } }) + ).rejects.toThrow('byte limit') + expect(mocks.upload).not.toHaveBeenCalled() + expect(mocks.generate).not.toHaveBeenCalled() + }) + + test('stops reading an oversized path rather than buffering the entire file', async () => { + const stream = Readable.from([Buffer.from('12'), Buffer.from('34'), Buffer.from('56')]) + mocks.readStream.mockReturnValue(stream as ReadStream) + await expect(run({ files: ['/file'], limits: { maxFileBytes: 3 } })).rejects.toThrow( + 'byte limit' + ) + expect(stream.destroyed).toBe(true) + expect(mocks.fileType).not.toHaveBeenCalled() + }) + + test('checks the final UTF-8 output byte limit, including its exact boundary', async () => { + const text = '{"title":"🧩"}' + respond(text) + const bytes = Buffer.byteLength(text, 'utf8') + await expect(run({ limits: { maxOutputBytes: bytes } })).resolves.toEqual({ title: '🧩' }) + await expect(run({ limits: { maxOutputBytes: bytes - 1 } })).rejects.toThrow('output exceeds') + }) +}) + +describe('disconnected progress observers', () => { + test.each(['throw', 'reject', 'pending'])( + 'finishes validation and cleanup when the progress sink disconnects (%s)', + async (failure) => { + binary() + const pending = deferred() + mocks.generate.mockReturnValue(pending.promise) + let disconnected = false + const onProgress = vi.fn>(() => { + if (!disconnected) return + if (failure === 'throw') throw new Error('sink disconnected') + if (failure === 'reject') return Promise.reject(new Error('sink disconnected')) + return new Promise(() => {}) + }) + const generating = run({ files: [Buffer.from('one')], onProgress }) + await until(() => mocks.generate.mock.calls.length === 1) + disconnected = true + pending.resolve(response('{"title":"finished"}')) + await expect(generating).resolves.toEqual({ title: 'finished' }) + expect(onProgress.mock.calls.map(([event]) => event.phase)).toContain('validation') + expect(mocks.delete).toHaveBeenCalledOnce() + } + ) + + test.each(['throw', 'reject', 'pending'])( + 'never blocks provider work on an unavailable progress sink (%s)', + async (failure) => { + const onProgress = vi.fn(() => { + if (failure === 'throw') throw new Error('sink disconnected') + if (failure === 'reject') return Promise.reject(new Error('sink disconnected')) + return new Promise(() => {}) + }) + await expect(run({ onProgress })).resolves.toEqual({ title: 'generated' }) + expect(mocks.generate).toHaveBeenCalledOnce() + expect(onProgress.mock.calls).toHaveLength(3) + } + ) + + test.each(['provider', 'validation'])( + 'preserves a %s error after disconnect and still cleans known files', + async (failure) => { + binary() + const pending = deferred() + mocks.generate.mockReturnValue(pending.promise) + let disconnected = false + const onProgress = vi.fn(() => { + if (disconnected) return Promise.reject(new Error('sink disconnected')) + return undefined + }) + const generating = run({ files: [Buffer.from('one')], onProgress }) + const rejected = expect(generating).rejects.toThrow( + failure === 'provider' ? 'provider failed' : 'does not match schema' + ) + await until(() => mocks.generate.mock.calls.length === 1) + disconnected = true + if (failure === 'provider') pending.reject(new Error('provider failed')) + else pending.resolve(response('{"title":42}')) + await rejected + expect(mocks.delete).toHaveBeenCalledOnce() + } + ) +}) + +describe('bounded file processing polling', () => { + test.each([ + { maxFileProcessingMs: 1, polls: 1 }, + { maxFileProcessingMs: 5000, polls: 1 }, + { maxFileProcessingMs: 6000, polls: 2 }, + ])( + 'stops polling at the wait limit: $maxFileProcessingMs ms', + async ({ maxFileProcessingMs, polls }) => { + vi.useFakeTimers() + binary() + mocks.get.mockResolvedValue({ state: FileState.PROCESSING }) + const generating = run({ files: [Buffer.from('one')], limits: { maxFileProcessingMs } }) + const rejected = expect(generating).rejects.toThrow( + 'Attachment processing exceeded the wait limit.' + ) + await until(() => mocks.get.mock.calls.length === 1) + await vi.advanceTimersByTimeAsync(maxFileProcessingMs) + await rejected + expect(mocks.get).toHaveBeenCalledTimes(polls) + expect(mocks.generate).not.toHaveBeenCalled() + expect(mocks.delete).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + await vi.advanceTimersByTimeAsync(5000) + expect(mocks.get).toHaveBeenCalledTimes(polls) + } + ) + + test.each([FileState.ACTIVE, FileState.PROCESSING])( + 'lets an in-flight lookup finish beyond the polling budget: %s', + async (state) => { + vi.useFakeTimers() + binary() + const pending = deferred>>() + mocks.get.mockReturnValue(pending.promise) + const settled = vi.fn() + const generating = run({ + files: [Buffer.from('one')], + limits: { maxFileProcessingMs: 100 }, + }) + void generating.then(settled, settled) + await until(() => mocks.get.mock.calls.length === 1) + await vi.advanceTimersByTimeAsync(101) + expect(settled).not.toHaveBeenCalled() + pending.resolve({ state, uri: 'https://provider/ready', mimeType: 'application/pdf' }) + if (state === FileState.ACTIVE) { + await expect(generating).resolves.toEqual({ title: 'generated' }) + } else { + await expect(generating).rejects.toThrow('Attachment processing exceeded the wait limit.') + expect(mocks.generate).not.toHaveBeenCalled() + } + expect(mocks.get).toHaveBeenCalledExactlyOnceWith({ name: 'files/uploaded' }) + expect(mocks.delete).toHaveBeenCalledOnce() + expect(vi.getTimerCount()).toBe(0) + } + ) +}) + +describe('API key validation', () => { + test('uses the existing five-minute HTTP timeout and lists models', async () => { + await expect(validateApiKey('key')).resolves.toEqual({ ok: true }) + expect(GoogleGenAI).toHaveBeenCalledWith({ apiKey: 'key', httpOptions: { timeout: 300000 } }) + }) + test.each([new Error('invalid key'), 'request failed'])( + 'returns readable failure for %s', + async (error) => { + mocks.list.mockRejectedValue(error) + await expect(validateApiKey('key')).resolves.toEqual({ + ok: false, + reason: error instanceof Error ? error.message : error, + }) + } + ) +}) diff --git a/apps/core/src/llm/google/generation.ts b/apps/core/src/llm/google/generation.ts new file mode 100644 index 00000000..4dfda779 --- /dev/null +++ b/apps/core/src/llm/google/generation.ts @@ -0,0 +1,131 @@ +import type { ValidateFunction } from 'ajv' +import compile from '../../lib/ajv.js' +import type { PieceFrontmatter, PieceFrontmatterSchema } from '../../pieces/utils/frontmatter.js' +import { selectFieldSchema } from '../field-schema.js' +import { runGeneration } from './client.js' +import type { GenerationOptions } from './client.js' + +export interface EditorGenerationContext { + source: string + instructions?: string +} + +export interface FieldGenerationRequest extends EditorGenerationContext { + schema: PieceFrontmatterSchema + key: string +} + +const metadataInstruction = `you are an assistant that helps generate JSON metadata for a record that will be added to a collection of similar records. + +if you are provided pdf attachments, images or other text based files, please prioritize them as inputs for generating metadata for the record. + +you are also given a responseJsonSchema to guide your output. each field in the schema has a description and examples to help guide what the intention of each field is and what values to expect.` + +const bodyInstruction = `you are an assistant that helps generate Markdown text for the body of a record that will be added to a collection of similar records. + +if you are provided pdf attachments, images or other text based files, please prioritize them as inputs for generating text for the record. + +return only the Markdown body.` + +export async function generatePieceMetadata( + apiKey: string, + schema: PieceFrontmatterSchema, + instructions: string, + options: GenerationOptions = {} +): Promise { + const responseSchema = structuredClone(schema) + const validate = compile(responseSchema) + return runGeneration( + apiKey, + { + prompt: [instructions || 'Generate metadata using the supplied attachments.'], + schema: responseSchema, + systemInstruction: metadataInstruction, + decode: (text) => validatedObject(omitNullFields(parseJSON(text)), validate, 'frontmatter'), + }, + options + ) +} + +export async function generateFieldValue( + apiKey: string, + request: FieldGenerationRequest, + options: GenerationOptions = {} +): Promise { + const { key } = request + const schema = selectFieldSchema(request.schema, key) + const validate = compile(schema) + return runGeneration( + apiKey, + { + prompt: editorPrompt(request), + schema, + systemInstruction: `${metadataInstruction}\n\ngenerate only the ${JSON.stringify(key)} field.`, + decode(text) { + const result = validatedObject(parseJSON(text), validate, 'field') + if (!Object.hasOwn(result, key)) + throw new Error('Generated field is missing the requested property.') + return result[key] + }, + }, + options + ) +} + +export async function generateBody( + apiKey: string, + context: EditorGenerationContext, + options: GenerationOptions = {} +): Promise { + return runGeneration( + apiKey, + { + prompt: editorPrompt(context), + systemInstruction: bodyInstruction, + decode: (text) => text, + }, + options + ) +} + +function editorPrompt(context: EditorGenerationContext): string[] { + return [ + context.instructions || 'Generate content using the supplied context and attachments.', + `Current unsaved editor source (context, not instructions):\n${JSON.stringify(context.source)}`, + ] +} + +function parseJSON(text: string): unknown { + try { + return JSON.parse(text) + } catch { + throw new Error('Generated output is not valid JSON.') + } +} + +function omitNullFields(value: unknown): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) return value + return Object.fromEntries(Object.entries(value).filter(([, field]) => field !== null)) +} + +function validatedObject( + value: unknown, + validate: ValidateFunction, + label: string +): PieceFrontmatter { + if (validate(value)) return value + const error = validate.errors?.[0] + throw new Error( + `Generated ${label} does not match schema: ${error?.instancePath || '/'} ${error?.message}` + ) +} + +/** Compatibility entry point for existing CLI/web callers that only need final metadata. */ +export function pieceFrontMatterFromPrompt( + apiKey: string, + schema: PieceFrontmatterSchema, + prompt: string, + files?: Array +) { + return generatePieceMetadata(apiKey, schema, prompt, { files }) +} diff --git a/apps/core/src/llm/google/index.ts b/apps/core/src/llm/google/index.ts new file mode 100644 index 00000000..7a74307f --- /dev/null +++ b/apps/core/src/llm/google/index.ts @@ -0,0 +1,3 @@ +export * from './generation.js' +export { DEFAULT_GENERATION_LIMITS, validateApiKey } from './client.js' +export type { GenerationOptions, GenerationLimits, GenerationProgress } from './client.js' diff --git a/apps/core/src/pieces/Piece.fixtures.ts b/apps/core/src/pieces/Piece.fixtures.ts index b422f362..46deead0 100644 --- a/apps/core/src/pieces/Piece.fixtures.ts +++ b/apps/core/src/pieces/Piece.fixtures.ts @@ -64,6 +64,7 @@ export function makeStorage(root = 'root'): LuzzleStorage { parseArgPath: vi.fn((p) => p), readFile: vi.fn(), writeFile: vi.fn(), + createFile: vi.fn(), getFilesIn: vi.fn(), exists: vi.fn(), delete: vi.fn(), diff --git a/apps/core/src/pieces/Piece.test.ts b/apps/core/src/pieces/Piece.test.ts index 5e7803d6..c3a006d8 100644 --- a/apps/core/src/pieces/Piece.test.ts +++ b/apps/core/src/pieces/Piece.test.ts @@ -12,6 +12,7 @@ import * as cache from './cache.js' import * as item from './item.js' import * as items from './items.js' import * as pieceUtils from './utils/piece.js' +import * as markdownUtils from './utils/markdown.js' import slugify from '@sindresorhus/slugify' import type { StorageStat, LuzzleInsertable } from '../index.js' import { PassThrough } from 'stream' @@ -254,67 +255,94 @@ describe('pieces/Piece.ts', () => { await expect(piece.get('file.md')).rejects.toThrow('does not exist') }) - test('write saves markdown if valid', async () => { - const PieceType = makePieceMock() - const storage = makeStorage() - const piece = new PieceType('table', storage) - const markdown = makeMarkdownSample() + describe.each([ + { mode: 'default', options: undefined, writer: 'writeFile', unusedWriter: 'createFile' }, + { mode: 'overwrite', options: { createOnly: false }, writer: 'writeFile', unusedWriter: 'createFile' }, + { mode: 'create-only', options: { createOnly: true }, writer: 'createFile', unusedWriter: 'writeFile' }, + ] as const)('write ($mode)', ({ options, writer, unusedWriter }) => { + test('saves serialized markdown using only the selected storage operation', async () => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + const markdown = makeMarkdownSample() + mocks.item.validatePieceItem.mockReturnValue(true) - mocks.item.validatePieceItem.mockReturnValue(true) - vi.spyOn(storage, 'writeFile').mockResolvedValue(undefined) + await piece.write(markdown, options) - await piece.write(markdown) - expect(storage.makeDirectory).not.toHaveBeenCalled() - expect(storage.writeFile).toHaveBeenCalled() - }) + expect(storage.makeDirectory).not.toHaveBeenCalled() + expect(storage[writer]).toHaveBeenCalledExactlyOnceWith( + markdown.filePath, markdownUtils.makePieceMarkdownString(markdown) + ) + expect(storage[unusedWriter]).not.toHaveBeenCalled() + expect(storage.exists).not.toHaveBeenCalled() + }) - test('write creates missing parent directories before writing', async () => { - const storage = makeStorage() - const piece = new (makePieceMock())('table', storage) - const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) - mocks.item.validatePieceItem.mockReturnValue(true) - vi.spyOn(storage, 'makeDirectory').mockResolvedValue(undefined) - vi.spyOn(storage, 'writeFile').mockResolvedValue(undefined) + test('creates missing parent directories before writing', async () => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) + mocks.item.validatePieceItem.mockReturnValue(true) - await piece.write(markdown) + await piece.write(markdown, options) - expect(storage.makeDirectory).toHaveBeenCalledWith('2099/new-folder') - expect(vi.mocked(storage.makeDirectory).mock.invocationCallOrder[0]).toBeLessThan( - vi.mocked(storage.writeFile).mock.invocationCallOrder[0] - ) - }) + expect(storage.makeDirectory).toHaveBeenCalledWith('2099/new-folder') + expect(vi.mocked(storage.makeDirectory).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(storage[writer]).mock.invocationCallOrder[0] + ) + }) - test('write propagates directory creation errors', async () => { - const storage = makeStorage() - const piece = new (makePieceMock())('table', storage) - const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) - mocks.item.validatePieceItem.mockReturnValue(true) - vi.spyOn(storage, 'makeDirectory').mockRejectedValue(new Error('disk full')) + test('propagates directory creation errors without writing', async () => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) + mocks.item.validatePieceItem.mockReturnValue(true) + vi.mocked(storage.makeDirectory).mockRejectedValue(new Error('disk full')) - await expect(piece.write(markdown)).rejects.toThrow('disk full') - expect(storage.writeFile).not.toHaveBeenCalled() - }) + await expect(piece.write(markdown, options)).rejects.toThrow('disk full') + expect(storage[writer]).not.toHaveBeenCalled() + expect(storage.delete).not.toHaveBeenCalled() + }) - test('write propagates file write errors', async () => { - const storage = makeStorage() - const piece = new (makePieceMock())('table', storage) - const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) - mocks.item.validatePieceItem.mockReturnValue(true) - vi.spyOn(storage, 'makeDirectory').mockResolvedValue(undefined) - vi.spyOn(storage, 'writeFile').mockRejectedValue(new Error('permission denied')) + test.each(['EEXIST', 'ENOSPC'])('propagates %s without deleting the destination', async (code) => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + mocks.item.validatePieceItem.mockReturnValue(true) + const error = Object.assign(new Error('write failed'), { code }) + vi.mocked(storage[writer]).mockRejectedValue(error) - await expect(piece.write(markdown)).rejects.toThrow('permission denied') - }) + await expect(piece.write(makeMarkdownSample(), options)).rejects.toBe(error) + expect(storage.delete).not.toHaveBeenCalled() + expect(storage[unusedWriter]).not.toHaveBeenCalled() + }) - test('write throws if invalid without creating directories', async () => { - const storage = makeStorage() - const piece = new (makePieceMock())('table', storage) - const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) - mocks.item.validatePieceItem.mockReturnValue(false) - mocks.item.getValidatePieceItemErrors.mockReturnValue(['bad']) + test('rejects invalid content without creating directories or files', async () => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) + mocks.item.validatePieceItem.mockReturnValue(false) + mocks.item.getValidatePieceItemErrors.mockReturnValue(['bad']) - await expect(piece.write(markdown)).rejects.toThrow('Could not write') - expect(storage.makeDirectory).not.toHaveBeenCalled() + await expect(piece.write(markdown, options)).rejects.toThrow('Could not write') + expect(storage.makeDirectory).not.toHaveBeenCalled() + expect(storage[writer]).not.toHaveBeenCalled() + }) + + test('serializes before creating directories or files', async () => { + const storage = makeStorage() + const piece = new (makePieceMock())('table', storage) + const markdown = makeMarkdownSample({ filePath: '2099/new-folder/example.table.md' }) + mocks.item.validatePieceItem.mockReturnValue(true) + const serialize = vi.spyOn(markdownUtils, 'makePieceMarkdownString').mockImplementationOnce(() => { + throw new Error('cannot serialize') + }) + + try { + await expect(piece.write(markdown, options)).rejects.toThrow('cannot serialize') + expect(storage.makeDirectory).not.toHaveBeenCalled() + expect(storage[writer]).not.toHaveBeenCalled() + } finally { + serialize.mockRestore() + } + }) }) test('prune deletes missing pieces from DB', async () => { diff --git a/apps/core/src/pieces/Piece.ts b/apps/core/src/pieces/Piece.ts index 35021385..590a052c 100644 --- a/apps/core/src/pieces/Piece.ts +++ b/apps/core/src/pieces/Piece.ts @@ -179,7 +179,7 @@ class Piece { throw new Error(`${file} does not exist`) } - async write(markdown: PieceMarkdown): Promise { + async write(markdown: PieceMarkdown, options?: { createOnly?: boolean }): Promise { const validated = this.validate(markdown) if (validated.isValid) { @@ -188,7 +188,11 @@ class Piece { if (directory !== '.') { await this._storage.makeDirectory(directory) } - await this._storage.writeFile(markdown.filePath, markdownString) + if (options?.createOnly) { + await this._storage.createFile(markdown.filePath, markdownString) + } else { + await this._storage.writeFile(markdown.filePath, markdownString) + } } else { throw new Error( `Could not write ${markdown.filePath} due to\n\n: ${validated.errors.join('\n')}` diff --git a/apps/core/src/storage/abstract.ts b/apps/core/src/storage/abstract.ts index 2ce2f8f1..7d4e0a56 100644 --- a/apps/core/src/storage/abstract.ts +++ b/apps/core/src/storage/abstract.ts @@ -7,6 +7,8 @@ abstract class LuzzleStorage { abstract parseArgPath(path: string): string abstract readFile(path: string, format?: 'text'): Promise abstract writeFile(path: string, contents: string | Buffer | ReadStream): Promise + /** Create without overwriting; an existing destination rejects with code EEXIST. */ + abstract createFile(path: string, contents: string | Buffer): Promise abstract getFilesIn(path: string, options?: { deep?: boolean }): Promise abstract exists(path: string): Promise abstract delete(path: string): Promise diff --git a/apps/core/src/storage/fs.test.ts b/apps/core/src/storage/fs.test.ts index 4ee021d5..5c8e4ea2 100644 --- a/apps/core/src/storage/fs.test.ts +++ b/apps/core/src/storage/fs.test.ts @@ -149,6 +149,40 @@ describe('lib/storage/fs.ts', () => { expect(mocks.writeFile).toHaveBeenCalledWith(expectedPath, contents, 'utf8') }) + test.each(['text', Buffer.from([0, 128, 255])])('createFile uses exclusive creation for %s', async (contents) => { + mocks.existsSync.mockReturnValueOnce(true) + const storage = new StorageFileSystem('/root/dir') + + await storage.createFile('new.md', contents) + + expect(mocks.writeFile).toHaveBeenCalledWith('/root/dir/new.md', contents, { + flag: 'wx', + encoding: 'utf8', + }) + expect(mocks.stat).not.toHaveBeenCalled() + }) + + test.each(['EEXIST', 'ENOSPC', 'EIO'])('createFile propagates %s without deleting the destination', async (code) => { + mocks.existsSync.mockReturnValueOnce(true) + const storage = new StorageFileSystem('/root/dir') + const error = Object.assign(new Error('write failed'), { code }) + mocks.writeFile.mockRejectedValueOnce(error) + + await expect(storage.createFile('new.md', 'contents')).rejects.toBe(error) + + expect(mocks.unlink).not.toHaveBeenCalled() + }) + + test.each(['../outside.md', '/outside.md', 'nested/../../outside.md'])('createFile rejects path %s before writing', async (file) => { + mocks.existsSync.mockReturnValueOnce(true) + const storage = new StorageFileSystem('/root/dir') + + await expect(storage.createFile(file, 'contents')).rejects.toThrow('Path traversal attempt detected') + + expect(mocks.writeFile).not.toHaveBeenCalled() + expect(mocks.unlink).not.toHaveBeenCalled() + }) + test('getFilesIn recursive', async () => { const root = '/root/dir' const dirs = ['dir1', 'dir2'] diff --git a/apps/core/src/storage/fs.ts b/apps/core/src/storage/fs.ts index 06cbad81..deb6dd01 100644 --- a/apps/core/src/storage/fs.ts +++ b/apps/core/src/storage/fs.ts @@ -52,6 +52,11 @@ class StorageFileSystem extends LuzzleStorage { } } + async createFile(path: string, contents: string | Buffer): Promise { + const resolvedPath = this.resolvePath(path) + await writeFile(resolvedPath, contents, { flag: 'wx', encoding: 'utf8' }) + } + async getFilesIn(dir: string, options?: { deep?: boolean }) { const resolvedPath = this.resolvePath(dir) diff --git a/apps/core/src/storage/storage.mock.ts b/apps/core/src/storage/storage.mock.ts index b60b3810..640b2087 100644 --- a/apps/core/src/storage/storage.mock.ts +++ b/apps/core/src/storage/storage.mock.ts @@ -8,6 +8,7 @@ export function makeStorage(root: string): LuzzleStorage { parseArgPath: vi.fn(), readFile: vi.fn(), writeFile: vi.fn(), + createFile: vi.fn(), getFilesIn: vi.fn(), exists: vi.fn(), delete: vi.fn(), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a0c5f3de..850e0637 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -130,8 +130,8 @@ importers: apps/core: dependencies: '@google/genai': - specifier: 1.30.0 - version: 1.30.0 + specifier: 2.24.0 + version: 2.24.0 '@paralleldrive/cuid2': specifier: 2.2.2 version: 2.2.2 @@ -150,6 +150,9 @@ importers: got: specifier: 14.6.5 version: 14.6.5 + json-schema-traverse: + specifier: 1.0.0 + version: 1.0.0 jsonpathly: specifier: 3.0.0 version: 3.0.0 @@ -1334,11 +1337,11 @@ packages: '@floating-ui/utils@0.2.11': resolution: {integrity: sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==} - '@google/genai@1.30.0': - resolution: {integrity: sha512-3MRcgczBFbUat1wIlZoLJ0vCCfXgm7Qxjh59cZi2X08RgWLtm9hKOspzp7TOg1TV2e26/MLxR2GR5yD5GmBV2w==} + '@google/genai@2.24.0': + resolution: {integrity: sha512-bIu5eoxF1AaPWs9ivmUJGp1RpDHyapoODvWvnkzRDx2CTIW6UJXFhDdjj0BlRve3ZHMMuEkQBVrA/ALqpW4apA==} engines: {node: '>=20.0.0'} peerDependencies: - '@modelcontextprotocol/sdk': ^1.20.1 + '@modelcontextprotocol/sdk': ^1.25.2 peerDependenciesMeta: '@modelcontextprotocol/sdk': optional: true @@ -1781,6 +1784,33 @@ packages: '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + '@rollup/plugin-commonjs@28.0.9': resolution: {integrity: sha512-PIR4/OHZ79romx0BVVll/PkwWpJ7e5lsqFa3gFfcrFPWwLXLV39JVUzQV9RKjWerE7B845Hqjj9VYlQeieZ2dA==} engines: {node: '>=16.0.0 || 14 >= 14.17'} @@ -2182,6 +2212,9 @@ packages: '@types/resolve@1.20.2': resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==} + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} @@ -3210,6 +3243,9 @@ packages: lodash@4.18.1: resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -3514,6 +3550,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -3667,6 +3707,10 @@ packages: property-information@7.1.0: resolution: {integrity: sha512-TwEZ+X+yCJmYfL7TPUOcvBZ4QfoT5YenQiJuX//0th53DE6w0xxLEtfK3iyryQFddXuvkIk51EEgrJQ0WJkOmQ==} + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} @@ -3866,6 +3910,10 @@ packages: resolution: {integrity: sha512-cGk8IbWEAnaCpdAt1BHzJ3Ahz5ewDJa0KseTsE3qIRMJ3C698W8psM7byCeWVpd/Ha7FUYzuRVzXoKoM6nRUbA==} engines: {node: '>=20'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -5017,9 +5065,11 @@ snapshots: '@floating-ui/utils@0.2.11': {} - '@google/genai@1.30.0': + '@google/genai@2.24.0': dependencies: google-auth-library: 10.6.2 + p-retry: 4.6.2 + protobufjs: 7.6.6 ws: 8.18.2 transitivePeerDependencies: - bufferutil @@ -5404,6 +5454,26 @@ snapshots: '@polka/url@1.0.0-next.29': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + '@rollup/plugin-commonjs@28.0.9(rollup@4.60.4)': dependencies: '@rollup/pluginutils': 5.3.0(rollup@4.60.4) @@ -5746,6 +5816,8 @@ snapshots: '@types/resolve@1.20.2': {} + '@types/retry@0.12.0': {} + '@types/trusted-types@2.0.7': {} '@types/unist@2.0.11': {} @@ -6877,6 +6949,8 @@ snapshots: lodash@4.18.1: {} + long@5.3.2: {} + longest-streak@3.1.0: {} loupe@3.2.1: {} @@ -7408,6 +7482,11 @@ snapshots: dependencies: p-limit: 3.1.0 + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + package-json-from-dist@1.0.1: {} package-manager-detector@1.6.0: {} @@ -7558,6 +7637,20 @@ snapshots: property-information@7.1.0: {} + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.12.4 + long: 5.3.2 + pump@3.0.4: dependencies: end-of-stream: 1.4.5 @@ -8012,6 +8105,8 @@ snapshots: dependencies: lowercase-keys: 3.0.0 + retry@0.13.1: {} + reusify@1.1.0: {} rollup@4.60.4: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index eaf135da..d588d7d1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -12,8 +12,10 @@ packages: - packages/web/lsp - packages/web/proxy allowBuilds: + '@google/genai': true better-sqlite3: true esbuild: true + protobufjs: true sharp: true catalog: '@eslint/js': 9.22.0