diff --git a/cli/src/modules/common/generatedImages.test.ts b/cli/src/modules/common/generatedImages.test.ts index 2baaae1b..4f465b19 100644 --- a/cli/src/modules/common/generatedImages.test.ts +++ b/cli/src/modules/common/generatedImages.test.ts @@ -170,9 +170,9 @@ describe('generatedImages', () => { clearGeneratedImages() }) - it('registers images from ACP base64 image blocks after MIME sniffing', async () => { + it('registers images from ACP base64 image blocks synchronously after MIME sniffing', () => { const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]) - const image = await registerGeneratedImageFromAcpBlock({ + const image = registerGeneratedImageFromAcpBlock({ type: 'image', mimeType: 'image/png', data: pngHeader.toString('base64') @@ -186,11 +186,11 @@ describe('generatedImages', () => { it('rejects oversized base64 before allocating a decoded buffer', async () => { const oversized = 'A'.repeat(MAX_GENERATED_IMAGE_BASE64_CHARS + 1) expect(decodeGeneratedImageBase64(oversized)).toBeNull() - await expect(registerGeneratedImageFromAcpBlock({ + expect(registerGeneratedImageFromAcpBlock({ type: 'image', mimeType: 'image/png', data: oversized, - })).resolves.toBeNull() + })).toBeNull() }) it('ignores URI-only ACP image blocks that would read local disk without a permission prompt', async () => { @@ -199,15 +199,15 @@ describe('generatedImages', () => { const path = join(dir, 'secret.png') writeFileSync(path, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00])) - await expect(registerGeneratedImageFromAcpBlock({ + expect(registerGeneratedImageFromAcpBlock({ type: 'image', uri: `file://${path}` - })).resolves.toBeNull() + })).toBeNull() - await expect(registerGeneratedImageFromAcpBlock({ + expect(registerGeneratedImageFromAcpBlock({ type: 'image', url: path - })).resolves.toBeNull() + })).toBeNull() }) it('registers images from local file paths in ACP uri blocks', async () => { @@ -250,7 +250,7 @@ describe('generatedImages', () => { expect(safeAcpFileName('file:///tmp/photos/icon.png')).toBe('icon.png') const pngHeader = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x00]) - const image = await registerGeneratedImageFromAcpBlock({ + const image = registerGeneratedImageFromAcpBlock({ type: 'image', mimeType: 'image/png', data: pngHeader.toString('base64'), diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts index 76767506..9ad33178 100644 --- a/cli/src/modules/common/generatedImages.ts +++ b/cli/src/modules/common/generatedImages.ts @@ -415,7 +415,7 @@ export function safeAcpFileName(uri: string | undefined | null): string | null { return null } -export async function registerGeneratedImageFromAcpBlock(block: unknown): Promise { +export function registerGeneratedImageFromAcpBlock(block: unknown): GeneratedImageMetadata | null { if (!isObject(block) || block.type !== 'image') { return null } diff --git a/cli/src/pi/loopPublishOrder.test.ts b/cli/src/pi/loopPublishOrder.test.ts new file mode 100644 index 00000000..67a9b38b --- /dev/null +++ b/cli/src/pi/loopPublishOrder.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it, vi } from 'vitest'; +import { wireTransportEvents } from './loop'; +import type { PiTransport } from './piTransport'; +import { PiSession } from './session'; +import type { CodexMessage } from '@/agent/messageConverter'; + +/** 1x1 red PNG. */ +const ONE_PX_PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + +function createHarness() { + let listener: ((event: Record) => void) | null = null; + const order: string[] = []; + const messages: CodexMessage[] = []; + const transport = { + onEvent: vi.fn((handler: (event: Record) => void) => { listener = handler; }), + send: vi.fn(), + } as unknown as PiTransport; + const client = { + keepAlive: vi.fn(), + updateMetadata: vi.fn(), + emitMessagesConsumed: vi.fn(), + sendSessionEvent: vi.fn(), + updateAgentState: vi.fn(), + emitSessionReady: vi.fn(), + getMetadata: vi.fn(() => null), + rpcHandlerManager: { registerHandler: vi.fn() }, + }; + const session = new PiSession({ + api: {} as never, + client: { + ...client, + sendAgentMessage: vi.fn((msg: CodexMessage) => { + order.push(`agent:${msg.type}`); + messages.push(msg); + }), + sendSessionEvent: vi.fn((msg: { type: string }) => { order.push(`session:${msg.type}`); }), + } as never, + path: '/tmp/test', + logPath: '/tmp/test.log', + startedBy: 'terminal', + startingMode: 'local', + model: 'deepseek-v4-flash-0731', + }); + + const controller = wireTransportEvents(transport, session, []); + + return { + emit: (event: Record) => listener?.(event), + order, + messages, + controller, + flush: () => new Promise((resolve) => setTimeout(resolve, 0)), + }; +} + + +describe('Pi publish ordering (real converters)', () => { + it('publishes inline images before a same-chunk extension notification', () => { + const h = createHarness(); + h.emit({ + type: 'tool_execution_end', toolCallId: 'image', toolName: 'hapi_display_image', + result: { content: [{ type: 'image', data: ONE_PX_PNG, mimeType: 'image/png' }] }, + isError: false, + }); + h.emit({ type: 'extension_ui_request', id: 'notice', method: 'notify', message: 'Image ready', notifyType: 'info' }); + expect(h.order).toEqual(['agent:tool-call-result', 'agent:generated-image', 'session:message']); + }); + it('publishes a tool-execution result before a later retry notice from the same chunk', async () => { + const h = createHarness(); + + h.emit({ + type: 'tool_execution_end', + toolCallId: 'tc-1', + toolName: 'bash', + result: { content: [{ type: 'text', text: 'ok' }], details: {} }, + isError: false, + }); + h.emit({ type: 'auto_retry_start', attempt: 1, maxAttempts: 3, delayMs: 10, errorMessage: '429' }); + + await h.flush(); + + // The tool result must reach the chat before the lifecycle notice. + const resultIdx = h.order.indexOf('agent:tool-call-result'); + const noticeIdx = h.order.indexOf('session:message'); + expect(resultIdx).toBeGreaterThanOrEqual(0); + expect(noticeIdx).toBeGreaterThan(resultIdx); + }); + + it('keeps image cards in event order ahead of a compaction notice', async () => { + const h = createHarness(); + + h.emit({ + type: 'tool_execution_end', + toolCallId: 'tc-2', + toolName: 'hapi_display_image', + result: { + content: [ + { type: 'text', text: 'screenshot' }, + { type: 'image', data: ONE_PX_PNG, mimeType: 'image/png' }, + ], + details: {}, + }, + isError: false, + }); + h.emit({ type: 'compaction_start', reason: 'threshold' }); + + await h.flush(); + + const resultIdx = h.order.indexOf('agent:tool-call-result'); + const imageIdx = h.order.indexOf('agent:generated-image'); + const noticeIdx = h.order.indexOf('session:message'); + expect(resultIdx).toBeGreaterThanOrEqual(0); + expect(imageIdx).toBeGreaterThan(resultIdx); + expect(noticeIdx).toBeGreaterThan(imageIdx); + }); + + it('preserves assistant snapshots on both sides of an inline image result', async () => { + const h = createHarness(); + const text = (delta: string) => h.emit({ + type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta }, + }); + h.emit({ type: 'message_start' }); + text('before'); + h.emit({ type: 'message_end' }); + h.emit({ + type: 'tool_execution_end', toolCallId: 'image', toolName: 'hapi_display_image', + result: { content: [{ type: 'image', data: ONE_PX_PNG, mimeType: 'image/png' }] }, + isError: false, + }); + h.emit({ type: 'message_start' }); + text('after'); + h.emit({ type: 'message_end' }); + await h.flush(); + expect(h.messages.map((message) => message.type === 'message' ? message.message : message.type)) + .toEqual(['before', 'tool-call-result', 'generated-image', 'after']); + }); + + it('retains queued assistant deltas when the prompt is rejected in the same chunk', async () => { + const h = createHarness(); + h.controller.beginPromptLifecycle('prompt-rejected'); + h.emit({ type: 'message_start' }); + h.emit({ type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta: 'before' } }); + await h.flush(); + + h.emit({ type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta: ' rejection' } }); + h.emit({ type: 'response', command: 'prompt', id: 'prompt-rejected', success: false, error: 'rejected' }); + await h.flush(); + expect(h.messages.filter((message) => message.type === 'message').map((message) => message.message)) + .toEqual(['before', 'before rejection']); + }); + + it('drains queued deltas before flushing a closed or failed transport', async () => { + const h = createHarness(); + h.emit({ type: 'message_start' }); + h.emit({ type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta: 'before' } }); + await h.flush(); + + h.emit({ type: 'message_update', assistantMessageEvent: { type: 'text_delta', delta: ' close' } }); + await h.controller.flush(); + await h.flush(); + expect(h.messages.filter((message) => message.type === 'message').map((message) => message.message)) + .toEqual(['before', 'before close']); + }); + +}); diff --git a/cli/src/pi/piEventConverter.test.ts b/cli/src/pi/piEventConverter.test.ts index b0c6d30b..6a448628 100644 --- a/cli/src/pi/piEventConverter.test.ts +++ b/cli/src/pi/piEventConverter.test.ts @@ -2,7 +2,31 @@ import { describe, it, expect } from 'vitest'; import { convertPiCompactionUsage, convertPiEvent, convertPiTurnUsage } from './piEventConverter'; import type { PiAgentEvent } from './types'; +// 1x1 red PNG. +const ONE_PX_PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='; + describe('convertPiEvent', () => { + it('renders nested Pi tool-result images while preserving failure status', () => { + const result = convertPiEvent({ + type: 'tool_execution_end', + toolCallId: 'codemode-1/1', + parentToolCallId: 'codemode-1', + toolName: 'screenshot', + result: { content: [ + { type: 'text', text: 'Screenshot captured before failure' }, + { type: 'image', data: ONE_PX_PNG, mimeType: 'image/png' }, + ] }, + isError: true, + } as PiAgentEvent); + expect(result).toEqual([ + { type: 'tool_result', id: 'codemode-1/1', output: 'Screenshot captured before failure', status: 'failed' }, + expect.objectContaining({ + type: 'generated_image', + mimeType: 'image/png', + source: { ingress: 'tool_result', toolName: 'screenshot', toolCallId: 'codemode-1/1' }, + }), + ]); + }); it('should return empty for message_update with text_delta (accumulated in runPi)', () => { // The converter intentionally emits nothing for message_update // — runPi accumulates text/thinking deltas and flushes a single @@ -36,22 +60,6 @@ describe('convertPiEvent', () => { expect(result).toEqual([]); }); - it('should return empty array for message_update with start sub-type', () => { - const result = convertPiEvent({ - type: 'message_update', - assistantMessageEvent: { type: 'start' } - }); - expect(result).toEqual([]); - }); - - it('should return empty array for message_update with done sub-type', () => { - const result = convertPiEvent({ - type: 'message_update', - assistantMessageEvent: { type: 'done', reason: 'stop' } - }); - expect(result).toEqual([]); - }); - it('should return empty array for message_update without assistantMessageEvent', () => { const result = convertPiEvent({ type: 'message_update' }); expect(result).toEqual([]); @@ -126,6 +134,109 @@ describe('convertPiEvent', () => { } as never)).toEqual([]); }); + it('emits a generated-image message for an inline image block next to the text result', () => { + const result = convertPiEvent({ + type: 'tool_execution_end', + toolCallId: 'tc-1', + toolName: 'hapi_display_image', + result: { + content: [ + { type: 'text', text: 'Here is the screenshot.' }, + { type: 'image', data: ONE_PX_PNG, mimeType: 'image/png' }, + ], + details: {}, + }, + isError: false + }); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + type: 'tool_result', + id: 'tc-1', + output: 'Here is the screenshot.', + status: 'completed', + }); + expect(result[1]).toMatchObject({ + type: 'generated_image', + mimeType: 'image/png', + source: { ingress: 'tool_result', toolName: 'hapi_display_image', toolCallId: 'tc-1' }, + }); + expect(result[1]).toHaveProperty('imageId'); + expect(result[1]).toHaveProperty('fileName'); + }); + + it('joins multiple text blocks and skips a malformed image block', () => { + const result = convertPiEvent({ + type: 'tool_execution_end', + toolCallId: 'tc-2', + toolName: 'hapi_display_image', + result: { + content: [ + { type: 'text', text: 'a' }, + { type: 'image', data: 'not-base64!', mimeType: 'image/png' }, + { type: 'text', text: 'b' }, + ], + details: {}, + }, + isError: false + }); + + // Invalid image payload is dropped; the text blocks remain the result. + expect(result).toEqual([{ + type: 'tool_result', + id: 'tc-2', + output: 'a\nb', + status: 'completed', + }]); + }); + + it.each([ + { type: 'image', uri: 'file:///private/image.png' }, + { type: 'image', url: 'https://example.invalid/image.png' }, + { type: 'image', data: ONE_PX_PNG, mimeType: 'image/jpeg' }, + ])('drops image blocks outside the inline bytes/MIME boundary: %j', async (block) => { + const result = convertPiEvent({ + type: 'tool_execution_end', toolCallId: 'boundary', toolName: 'screenshot', isError: false, + result: { content: [{ type: 'text', text: 'kept' }, block] }, + }); + expect(result).toEqual([{ type: 'tool_result', id: 'boundary', output: 'kept', status: 'completed' }]); + }); + + it('keeps a structured result without images as the plain text output', () => { + const result = convertPiEvent({ + type: 'tool_execution_end', + toolCallId: 'tc-3', + toolName: 'bash', + result: { content: [{ type: 'text', text: 'ok' }], details: {} }, + isError: false + }); + + expect(result).toEqual([{ + type: 'tool_result', + id: 'tc-3', + output: 'ok', + status: 'completed', + }]); + }); + + it('falls back to the raw result when content is not a block array', () => { + const raw = { some: 'other', shape: true }; + const result = convertPiEvent({ + type: 'tool_execution_end', + toolCallId: 'tc-4', + toolName: 'bash', + result: raw, + isError: false + }); + + expect(result).toEqual([{ + type: 'tool_result', + id: 'tc-4', + output: raw, + status: 'completed', + }]); + }); + it('should defer turn usage and convert only turn completion', () => { const result = convertPiEvent({ type: 'turn_end', @@ -271,7 +382,6 @@ describe('convertPiEvent', () => { weird.type = 'message_update'; weird.assistantMessageEvent = undefined; // Should not throw - expect(() => convertPiEvent(weird as unknown as PiAgentEvent)).not.toThrow(); expect(convertPiEvent(weird as unknown as PiAgentEvent)).toEqual([]); }); }); diff --git a/cli/src/pi/piEventConverter.ts b/cli/src/pi/piEventConverter.ts index 1f9c1906..3d950c21 100644 --- a/cli/src/pi/piEventConverter.ts +++ b/cli/src/pi/piEventConverter.ts @@ -1,4 +1,5 @@ import { logger } from '@/ui/logger'; +import { registerGeneratedImageFromAcpBlock } from '@/modules/common/generatedImages'; import type { AgentMessage } from '@/agent/types'; import { PiToolExecutionEndEventSchema, @@ -41,6 +42,31 @@ export function convertPiCompactionUsage(estimatedTokensAfter: number | undefine }; } +/** + * Splits a structured Pi tool result ({ content: TextContent[] | ImageContent[], ... }) + * into displayable text and image blocks. Returns null for legacy string results + * so callers keep the pass-through behavior. + */ +export function splitPiToolResult(result: unknown): { text: string; images: unknown[] } | null { + if (!result || typeof result !== 'object' || !Array.isArray((result as { content?: unknown }).content)) { + return null; + } + const text: string[] = []; + const images: unknown[] = []; + for (const block of (result as { content: unknown[] }).content) { + if (!block || typeof block !== 'object') { + continue; + } + const record = block as Record; + if (record.type === 'text' && typeof record.text === 'string') { + text.push(record.text); + } else if (record.type === 'image') { + images.push(block); + } + } + return { text: text.join('\n'), images }; +} + /** Converts validated Pi lifecycle events to HAPI chat messages. */ export function convertPiEvent(event: PiAgentEvent): AgentMessage[] { switch (event.type) { @@ -70,12 +96,53 @@ export function convertPiEvent(event: PiAgentEvent): AgentMessage[] { case 'tool_execution_end': { const parsed = PiToolExecutionEndEventSchema.safeParse(event); if (!parsed.success) return []; - return [{ + + const { toolCallId, result, isError } = parsed.data; + const split = splitPiToolResult(result); + if (!split) { + return [{ + type: 'tool_result', + id: toolCallId, + output: result, + status: isError ? 'failed' : 'completed', + }]; + } + + const messages: AgentMessage[] = [{ type: 'tool_result', - id: parsed.data.toolCallId, - output: parsed.data.result, - status: parsed.data.isError ? 'failed' : 'completed', + id: toolCallId, + output: split.text, + status: isError ? 'failed' : 'completed', }]; + + // Inline image blocks (e.g. from the hapi_display_image extension or + // any tool that hands back ImageContent) land in the chat via the + // same generated-image pipeline the ACP backend already uses. + for (const block of split.images) { + try { + const image = registerGeneratedImageFromAcpBlock(block); + if (!image) { + continue; + } + messages.push({ + type: 'generated_image', + imageId: image.id, + fileName: image.fileName, + mimeType: image.mimeType, + source: { + ingress: 'tool_result', + toolName: parsed.data.toolName, + toolCallId, + }, + }); + } catch (error) { + logger.debug( + '[pi] Failed to register tool-result image:', + error instanceof Error ? error.message : String(error) + ); + } + } + return messages; } case 'turn_end': { const turn = event as PiTurnEndEvent; diff --git a/cli/src/pi/titleExtension.test.ts b/cli/src/pi/titleExtension.test.ts index 216e37a6..28d6b4a1 100644 --- a/cli/src/pi/titleExtension.test.ts +++ b/cli/src/pi/titleExtension.test.ts @@ -1,3 +1,4 @@ +import * as fs from 'node:fs/promises'; import { mkdtemp, readdir, readFile, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -124,3 +125,165 @@ describe('materializePiTitleExtension', () => { expect(later.systemPrompt).toContain('Rename only when'); }); }); + + +type ImageResult = { content: Array<{ type: string; text?: string; data?: string; mimeType?: string }>; isError?: boolean }; + +function loadImageTool(fileSystem: unknown = fs) { + // Execute the bundled source with only its external dependencies injected. + const load = new Function('Type', 'fs', PI_TITLE_EXTENSION_SOURCE + .replace("import { Type } from '@sinclair/typebox';", '') + .replace('export default function', 'function') + .replace("await import('node:fs/promises')", 'fs') + + '\nreturn hapiTitleExtension;'); + const registerTool = vi.fn(); + load({ Object: (properties: unknown) => properties, String: (options: unknown) => options }, fileSystem)({ + on: vi.fn(), registerTool, + }); + return registerTool.mock.calls.find(([tool]) => tool.name === 'hapi_display_image')![0] as { + execute: (id: string, params: { path: string }) => Promise; + }; +} + +describe('hapi_display_image execution', () => { + const png = Buffer.from('89504e470d0a1a0a', 'hex'); + const limit = 25 * 1024 * 1024; + + async function withImage(run: (path: string) => Promise) { + const dir = await mkdtemp(join(tmpdir(), 'hapi-pi-image-')); + try { + const path = join(dir, 'image.png'); + await writeFile(path, png); + await run(path); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + } + + it.each([ + ['png', png, 'image/png'], + ['jpeg', Buffer.from('ffd8ff', 'hex'), 'image/jpeg'], + ['gif', Buffer.from('GIF89a'), 'image/gif'], + ['webp', Buffer.from('RIFF0000WEBP'), 'image/webp'], + ] as const)('returns a base64 image block for %s magic bytes', async (_name, bytes, mimeType) => { + await withImage(async (path) => { + await writeFile(path, bytes); + expect(await loadImageTool().execute('image', { path })).toEqual({ + content: [{ type: 'image', data: bytes.toString('base64'), mimeType }], details: {}, + }); + }); + }); + + it('rejects unsupported content and incomplete PNG signatures', async () => { + await withImage(async (path) => { + for (const bytes of [Buffer.from('not an image'), Buffer.from('89504e4700000000', 'hex')]) { + await writeFile(path, bytes); + const result = await loadImageTool().execute('image', { path }); + expect(result.content[0]).toMatchObject({ type: 'text', text: expect.stringContaining('Unsupported') }); + } + }); + }); + + it('rejects missing, non-regular and oversized paths', async () => { + await withImage(async (path) => { + const tool = loadImageTool(); + expect((await tool.execute('image', { path: '' })).content[0].text).toContain('non-empty'); + expect((await tool.execute('image', { path: path + '.missing' })).content[0].text).toContain('Failed to read'); + expect((await tool.execute('image', { path: join(path, '..') })).content[0].type).toBe('text'); + await fs.truncate(path, limit + 1); + expect((await tool.execute('image', { path })).content[0].text).toContain('25 MiB'); + }); + }); + + it('marks display failures as errors for Pi tool execution', async () => { + expect(await loadImageTool().execute('image', { path: '' })).toMatchObject({ isError: true }); + }); + + it.each(['short reads', 'early EOF', 'shrinkage'])( + 'bounds descriptor reads and closes the descriptor after %s', + async (scenario) => { + const stat = vi.fn().mockResolvedValue({ size: png.length, isFile: () => true }); + if (scenario === 'shrinkage') { + stat.mockResolvedValueOnce({ size: png.length, isFile: () => true }) + .mockResolvedValue({ size: png.length - 1 }); + } + const close = vi.fn().mockResolvedValue(undefined); + const tool = loadImageTool({ + ...fs, + open: async () => ({ + stat, close, + read: async (buffer: Buffer, offset: number, length: number, position: number) => { + expect(buffer.length).toBe(png.length); + const bytesRead = scenario === 'early EOF' ? 0 : Math.min(2, length); + png.copy(buffer, offset, position, position + bytesRead); + return { bytesRead }; + }, + }), + }); + const result = await tool.execute('image', { path: '/image.png' }); + if (scenario === 'short reads') { + expect(result.content[0]).toEqual({ type: 'image', data: png.toString('base64'), mimeType: 'image/png' }); + } else { + expect(result.content[0].text).toContain('File changed while reading'); + } + expect(close).toHaveBeenCalledOnce(); + }, + ); + + it.skipIf(process.platform === 'win32')('rejects a device without reading it', async () => { + expect((await loadImageTool().execute('image', { path: '/dev/zero' })).content[0].text).toContain('not a regular file'); + }); + + it('bounds reads and rejects a file that grows after its size is checked', async () => { + await withImage(async (path) => { + let grown = false; + const grow = async () => { + if (!grown) { + grown = true; + await fs.truncate(path, limit + 1); + } + }; + const read = vi.fn(async (handle: fs.FileHandle, buffer: Buffer, offset: number, length: number, position: number) => { + expect(buffer.length).toBeLessThanOrEqual(limit); + return handle.read(buffer, offset, length, position); + }); + const tool = loadImageTool({ + ...fs, + stat: async () => { const info = await fs.stat(path); await grow(); return info; }, + open: async () => { + const handle = await fs.open(path, 'r'); + return { + stat: async () => { const info = await handle.stat(); await grow(); return info; }, + read: (buffer: Buffer, offset: number, length: number, position: number) => read(handle, buffer, offset, length, position), + close: () => handle.close(), + }; + }, + }); + const result = await tool.execute('image', { path }); + expect(result.content[0].type).toBe('text'); + expect(result.content[0].text).toContain('File changed while reading'); + expect(read).toHaveBeenCalled(); + }); + }); + + it('keeps reading the opened image if the pathname is replaced', async () => { + await withImage(async (path) => { + const replace = async () => { + await fs.rename(path, path + '.original'); + await writeFile(path, 'not an image'); + }; + const tool = loadImageTool({ + ...fs, + stat: async () => { const info = await fs.stat(path); await replace(); return info; }, + open: async () => { + const handle = await fs.open(path, 'r'); + await replace(); + return handle; + }, + }); + expect((await tool.execute('image', { path })).content[0]).toEqual({ + type: 'image', data: png.toString('base64'), mimeType: 'image/png', + }); + }); + }); +}); diff --git a/cli/src/pi/titleExtension.ts b/cli/src/pi/titleExtension.ts index 678fafba..7c05f99e 100644 --- a/cli/src/pi/titleExtension.ts +++ b/cli/src/pi/titleExtension.ts @@ -63,6 +63,61 @@ export default function hapiTitleExtension(pi) { return { content: [{ type: 'text', text: 'Session title set: ' + title }], details: {} }; } }); + + pi.registerTool({ + name: 'hapi_display_image', + label: 'Display Image', + promptSnippet: 'Show a local image to the user in the HAPI chat.', + description: "Display a local image file to the human user inline in the current HAPI chat session. Call it when the user asks to view a local image, screenshot, or diagram. The image is shown in the chat; note it also enters the model context, so prefer reading the file for analysis and this tool for presentation.", + parameters: Type.Object({ + path: Type.String({ description: 'Absolute filesystem path of the local image to display (png, jpeg, gif or webp, up to 25 MiB).' }) + }), + async execute(_toolCallId, params) { + const filePath = String((params && params.path) || '').trim(); + if (!filePath) { + return { content: [{ type: 'text', text: 'Error: path must be non-empty' }], details: {}, isError: true }; + } + const { constants, open } = await import('node:fs/promises'); + let bytes; + try { + // One descriptor and a fixed buffer keep replacement/growth bounded. + const handle = await open(filePath, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)); + try { + const meta = await handle.stat(); + if (!meta.isFile()) throw new Error('Path is not a regular file'); + if (meta.size > 25 * 1024 * 1024) throw new Error('File exceeds the 25 MiB display limit'); + bytes = Buffer.alloc(meta.size); + for (let offset = 0; offset < bytes.length;) { + const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset); + if (bytesRead === 0) throw new Error('File changed while reading'); + offset += bytesRead; + } + if ((await handle.stat()).size !== meta.size) throw new Error('File changed while reading'); + } finally { + await handle.close(); + } + } catch (error) { + const detail = error && error.message ? error.message : String(error); + return { content: [{ type: 'text', text: 'Failed to read file: ' + detail }], details: {}, isError: true }; + } + const mimeType = detectImageMime(bytes); + if (!mimeType) { + return { content: [{ type: 'text', text: 'Unsupported image content (png, jpeg, gif or webp only)' }], details: {}, isError: true }; + } + return { + content: [{ type: 'image', data: bytes.toString('base64'), mimeType: mimeType }], + details: {} + }; + } + }); +} + +function detectImageMime(bytes) { + if (bytes.length >= 8 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47 && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) return 'image/png'; + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'; + if (bytes.length >= 6 && bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x38 && (bytes[4] === 0x37 || bytes[4] === 0x39) && bytes[5] === 0x61) return 'image/gif'; + if (bytes.length >= 12 && bytes[0] === 0x52 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x46 && bytes[8] === 0x57 && bytes[9] === 0x45 && bytes[10] === 0x42 && bytes[11] === 0x50) return 'image/webp'; + return null; } `; diff --git a/docs/guide/agents.md b/docs/guide/agents.md index 1570c2a7..5500d269 100644 --- a/docs/guide/agents.md +++ b/docs/guide/agents.md @@ -290,7 +290,7 @@ overall permission policy. - **Kimi** (`hapi kimi`) — Moonshot AI's Kimi CLI over ACP (`kimi acp`). [MoonshotAI/kimi-cli](https://github.com/MoonshotAI/kimi-cli) - **OpenCode** (`hapi opencode`) — the open-source OpenCode agent over ACP (`opencode acp`). [opencode.ai](https://opencode.ai) - **Antigravity** (`hapi agy`) — Google's Antigravity CLI (`agy`), driven headlessly via print mode: every turn spawns `agy -p --conversation --output-format stream-json`, and NDJSON events (init / step_update / result) are streamed into the chat. There is no PTY/TUI wrapper and no hook-based permission bridge: permission handling uses agy's own `settings.json` allow/deny rules (`request-review`) or `--dangerously-skip-permissions` (`always-proceed`). Tool calls that lack an allow-rule are auto-denied by agy and surfaced as a chat hint. MCP servers are configured the standard agy way — in the user's global `~/.gemini/config/mcp_config.json` or a workspace `.agents/mcp_config.json` — and are loaded natively by agy in headless mode (no HAPI injection). Remote-only — there is no local terminal input path. [Google Antigravity](https://antigravity.google) -- **Pi** (`hapi pi`) — the Pi coding agent running as `pi --mode rpc` (JSON-line RPC over piped stdio); remote-control only, no local TUI input path. [badlogic/pi-mono](https://github.com/badlogic/pi-mono) +- **Pi** (`hapi pi`) — the Pi coding agent running as `pi --mode rpc` (JSON-line RPC over piped stdio); remote-control only, no local TUI input path. [earendil-works/pi](https://github.com/earendil-works/pi) HAPI translates a subset of Pi's TUI slash commands to native Pi RPC calls, so they work from the web chat as well: @@ -301,6 +301,8 @@ overall permission policy. Pi's extension commands and prompt templates (discovered via `get_commands`) keep working from the `/` menu, and skills are available through `$skill-name` like other ACP flavors. Other Pi TUI builtins (e.g. `/tree`, `/export`, `/reload`) cannot run over RPC; typing them in web shows an explicit "terminal-only" notice instead of silently forwarding the text to the model. + Pi sessions include a `hapi_display_image` tool for showing local PNG, JPEG, GIF, or WebP images up to 25 MiB in the chat. Images returned by Pi tools render inline even when the selected model has no vision support. The display tool also returns image bytes to Pi's model context. + ## Related - [How it Works](./how-it-works.md) - Architecture and data flow