diff --git a/cli/src/codex/codexRemoteLauncher.ts b/cli/src/codex/codexRemoteLauncher.ts index 986857bb..581ace5f 100644 --- a/cli/src/codex/codexRemoteLauncher.ts +++ b/cli/src/codex/codexRemoteLauncher.ts @@ -13,6 +13,7 @@ import type { CodexSession } from './session'; import type { EnhancedMode } from './loop'; import { hasCodexCliOverrides } from './utils/codexCliOverrides'; import { AppServerEventConverter } from './utils/appServerEventConverter'; +import { registerGeneratedImage } from '@/modules/common/generatedImages'; import { registerAppServerPermissionHandlers } from './utils/appServerPermissionAdapter'; import { buildThreadStartParams, buildTurnStartParams } from './utils/appServerConfig'; import type { ThreadGoal, ThreadGoalStatus } from './appServerTypes'; @@ -1968,6 +1969,28 @@ class CodexRemoteLauncher extends RemoteLauncherBase { }); } } + if (msgType === 'generated_image') { + const sourceImageId = asString(msg.image_id ?? msg.imageId ?? msg.id); + const imageId = randomUUID(); + const savedPath = asString(msg.saved_path ?? msg.savedPath); + if (savedPath) { + const image = registerGeneratedImage({ + id: imageId, + path: savedPath, + fileName: asString(msg.file_name ?? msg.fileName), + mimeType: asString(msg.mime_type ?? msg.mimeType) + }); + messageBuffer.addMessage(`Generated image: ${image.fileName}`, 'assistant'); + session.sendAgentMessage({ + type: 'generated-image', + imageId: image.id, + sourceImageId, + fileName: image.fileName, + mimeType: image.mimeType, + id: randomUUID() + }); + } + } if (msgType === 'exec_command_begin' || msgType === 'exec_approval_request') { const callId = asString(msg.call_id ?? msg.callId); if (callId) { diff --git a/cli/src/codex/utils/appServerEventConverter.test.ts b/cli/src/codex/utils/appServerEventConverter.test.ts index 88d0a7c4..6bb8d9c4 100644 --- a/cli/src/codex/utils/appServerEventConverter.test.ts +++ b/cli/src/codex/utils/appServerEventConverter.test.ts @@ -743,9 +743,8 @@ describe('AppServerEventConverter', () => { ]); }); - it('truncates large unhandled notification payloads before logging', () => { + it('converts completed image generation items without including large result payloads', () => { const converter = new AppServerEventConverter(); - const debug = vi.spyOn(logger, 'debug').mockImplementation(() => undefined); const largeImageResult = 'a'.repeat(4096); const events = converter.handleNotification('item/completed', { @@ -753,6 +752,31 @@ describe('AppServerEventConverter', () => { id: 'image-1', type: 'imageGeneration', result: largeImageResult, + savedPath: '/tmp/image.png', + mimeType: 'image/png' + } + }); + + expect(events).toEqual([{ + type: 'generated_image', + image_id: 'image-1', + saved_path: '/tmp/image.png', + file_name: 'image.png', + mime_type: 'image/png' + }]); + expect(JSON.stringify(events)).not.toContain(largeImageResult); + }); + + it('truncates large unhandled notification payloads before logging', () => { + const converter = new AppServerEventConverter(); + const debug = vi.spyOn(logger, 'debug').mockImplementation(() => undefined); + const largeResult = 'a'.repeat(4096); + + const events = converter.handleNotification('item/completed', { + item: { + id: 'unknown-1', + type: 'unknownLargePayload', + result: largeResult, savedPath: '/tmp/image.png' } }); @@ -760,7 +784,7 @@ describe('AppServerEventConverter', () => { expect(events).toEqual([]); expect(debug).toHaveBeenCalledTimes(1); const logged = debug.mock.calls[0]?.[1] as { params?: { item?: { result?: string; savedPath?: string } } }; - expect(logged.params?.item?.result).not.toBe(largeImageResult); + expect(logged.params?.item?.result).not.toBe(largeResult); expect(logged.params?.item?.result).toContain('[truncated 3584 chars for logs]'); expect(logged.params?.item?.savedPath).toBe('/tmp/image.png'); diff --git a/cli/src/codex/utils/appServerEventConverter.ts b/cli/src/codex/utils/appServerEventConverter.ts index 57094ed6..28bae2be 100644 --- a/cli/src/codex/utils/appServerEventConverter.ts +++ b/cli/src/codex/utils/appServerEventConverter.ts @@ -56,6 +56,28 @@ function extractCommand(value: unknown): string | null { return null; } +function extractGeneratedImagePath(item: Record): string | null { + return asString( + item.savedPath + ?? item.saved_path + ?? item.path + ?? item.filePath + ?? item.file_path + ?? item.outputPath + ?? item.output_path + ); +} + +function extractGeneratedImageMimeType(item: Record): string | null { + return asString(item.mimeType ?? item.mime_type ?? item.mediaType ?? item.media_type); +} + +function extractGeneratedImageFileName(item: Record, savedPath: string): string { + const direct = asString(item.fileName ?? item.file_name ?? item.filename ?? item.name); + if (direct) return direct; + return savedPath.split(/[\\/]/).filter(Boolean).pop() ?? 'generated-image.png'; +} + function extractChanges(value: unknown): Record | null { const record = asRecord(value); if (record) return record; @@ -871,6 +893,24 @@ export class AppServerEventConverter { return events; } + if (itemType === 'imagegeneration') { + if (method === 'item/completed') { + const savedPath = extractGeneratedImagePath(item); + if (!savedPath) { + logger.debug('[AppServerEventConverter] imageGeneration missing savedPath', sanitizeUnhandledNotificationLogValue({ item })); + return events; + } + events.push(scoped({ + type: 'generated_image', + image_id: itemId, + saved_path: savedPath, + file_name: extractGeneratedImageFileName(item, savedPath), + ...(extractGeneratedImageMimeType(item) ? { mime_type: extractGeneratedImageMimeType(item) } : {}) + })); + } + return events; + } + if (itemType === 'collabagenttoolcall') { const toolName = normalizeCollabAgentToolName(item.tool ?? item.name); if (!toolName) return events; diff --git a/cli/src/modules/common/generatedImages.ts b/cli/src/modules/common/generatedImages.ts new file mode 100644 index 00000000..39c45f20 --- /dev/null +++ b/cli/src/modules/common/generatedImages.ts @@ -0,0 +1,50 @@ +import { basename, extname } from 'path' + +export type GeneratedImageMetadata = { + id: string + path: string + fileName: string + mimeType: string + createdAt: number +} + +const IMAGE_MIME_BY_EXTENSION: Record = { + '.apng': 'image/apng', + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.gif': 'image/gif', + '.ico': 'image/x-icon', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.webp': 'image/webp' +} + +const generatedImages = new Map() + +export function resolveGeneratedImageMimeType(path: string): string { + return IMAGE_MIME_BY_EXTENSION[extname(path).toLowerCase()] ?? 'application/octet-stream' +} + +export function registerGeneratedImage(args: { id: string; path: string; mimeType?: string | null; fileName?: string | null }): GeneratedImageMetadata { + const metadata: GeneratedImageMetadata = { + id: args.id, + path: args.path, + fileName: args.fileName || basename(args.path) || `${args.id}.png`, + mimeType: args.mimeType || resolveGeneratedImageMimeType(args.path), + createdAt: Date.now() + } + generatedImages.set(args.id, metadata) + return metadata +} + +export function getGeneratedImage(id: string): GeneratedImageMetadata | null { + return generatedImages.get(id) ?? null +} + +export function clearGeneratedImages(): void { + generatedImages.clear() +} diff --git a/cli/src/modules/common/handlers/files.ts b/cli/src/modules/common/handlers/files.ts index 486d1acb..8cf8a5e8 100644 --- a/cli/src/modules/common/handlers/files.ts +++ b/cli/src/modules/common/handlers/files.ts @@ -4,6 +4,7 @@ import { createHash } from 'crypto' import { resolve } from 'path' import type { RpcHandlerManager } from '@/api/rpc/RpcHandlerManager' import { validatePath } from '../pathSecurity' +import { getGeneratedImage } from '../generatedImages' import { getErrorMessage, rpcError } from '../rpcResponses' interface ReadFileRequest { @@ -16,6 +17,18 @@ interface ReadFileResponse { error?: string } +interface ReadGeneratedImageRequest { + id: string +} + +interface ReadGeneratedImageResponse { + success: boolean + content?: string + mimeType?: string + fileName?: string + error?: string +} + interface WriteFileRequest { path: string content: string @@ -48,6 +61,28 @@ export function registerFileHandlers(rpcHandlerManager: RpcHandlerManager, worki } }) + rpcHandlerManager.registerHandler('readGeneratedImage', async (data) => { + logger.debug('Read generated image request:', data.id) + + const image = getGeneratedImage(data.id) + if (!image) { + return rpcError('Generated image not found') + } + + try { + const buffer = await readFile(image.path) + return { + success: true, + content: buffer.toString('base64'), + mimeType: image.mimeType, + fileName: image.fileName + } + } catch (error) { + logger.debug('Failed to read generated image:', error) + return rpcError(getErrorMessage(error, 'Failed to read generated image')) + } + }) + rpcHandlerManager.registerHandler('writeFile', async (data) => { logger.debug('Write file request:', data.path) diff --git a/hub/src/sync/rpcGateway.ts b/hub/src/sync/rpcGateway.ts index 34856e33..89643138 100644 --- a/hub/src/sync/rpcGateway.ts +++ b/hub/src/sync/rpcGateway.ts @@ -19,6 +19,14 @@ export type RpcReadFileResponse = { error?: string } +export type RpcGeneratedImageResponse = { + success: boolean + content?: string + mimeType?: string + fileName?: string + error?: string +} + export type RpcUploadFileResponse = { success: boolean path?: string @@ -229,6 +237,10 @@ export class RpcGateway { return await this.sessionRpc(sessionId, 'readFile', { path }) as RpcReadFileResponse } + async readGeneratedImage(sessionId: string, imageId: string): Promise { + return await this.sessionRpc(sessionId, 'readGeneratedImage', { id: imageId }) as RpcGeneratedImageResponse + } + async listDirectory(sessionId: string, path: string): Promise { return await this.sessionRpc(sessionId, 'listDirectory', { path }) as RpcListDirectoryResponse } diff --git a/hub/src/sync/syncEngine.ts b/hub/src/sync/syncEngine.ts index bc883607..872031bc 100644 --- a/hub/src/sync/syncEngine.ts +++ b/hub/src/sync/syncEngine.ts @@ -20,6 +20,7 @@ import { type RpcCodexModel, type RpcCommandResponse, type RpcDeleteUploadResponse, + type RpcGeneratedImageResponse, type RpcListDirectoryResponse, type RpcListCodexModelsResponse, type RpcListOpencodeModelsResponse, @@ -37,6 +38,7 @@ export type { RpcCodexModel, RpcCommandResponse, RpcDeleteUploadResponse, + RpcGeneratedImageResponse, RpcListDirectoryResponse, RpcListCodexModelsResponse, RpcListOpencodeModelsResponse, @@ -655,6 +657,10 @@ export class SyncEngine { return await this.rpcGateway.readSessionFile(sessionId, path) } + async readGeneratedImage(sessionId: string, imageId: string): Promise { + return await this.rpcGateway.readGeneratedImage(sessionId, imageId) + } + async listDirectory(sessionId: string, path: string): Promise { return await this.rpcGateway.listDirectory(sessionId, path) } diff --git a/hub/src/web/routes/git.ts b/hub/src/web/routes/git.ts index 972f6e36..8cd27a1e 100644 --- a/hub/src/web/routes/git.ts +++ b/hub/src/web/routes/git.ts @@ -17,6 +17,10 @@ const filePathSchema = z.object({ path: z.string().min(1) }) +const generatedImageSchema = z.object({ + imageId: z.string().min(1) +}) + function parseBooleanParam(value: string | undefined): boolean | undefined { if (value === 'true') return true if (value === 'false') return false @@ -130,6 +134,35 @@ export function createGitRoutes(getSyncEngine: () => SyncEngine | null): Hono { + const engine = requireSyncEngine(c, getSyncEngine) + if (engine instanceof Response) { + return engine + } + + const sessionResult = requireSessionFromParam(c, engine) + if (sessionResult instanceof Response) { + return sessionResult + } + + const parsed = generatedImageSchema.safeParse(c.req.param()) + if (!parsed.success) { + return c.json({ error: 'Invalid generated image id' }, 400) + } + + const result = await runRpc(() => engine.readGeneratedImage(sessionResult.sessionId, parsed.data.imageId)) + if (!result.success || !result.content) { + return c.json({ success: false, error: result.error ?? 'Generated image not found' }, 404) + } + + const bytes = Uint8Array.from(Buffer.from(result.content, 'base64')) + return c.body(bytes, 200, { + 'Content-Type': result.mimeType ?? 'application/octet-stream', + 'Content-Disposition': `inline; filename="${encodeURIComponent(result.fileName ?? 'generated-image')}"`, + 'Cache-Control': 'no-store' + }) + }) + app.get('/sessions/:id/files', async (c) => { const engine = requireSyncEngine(c, getSyncEngine) if (engine instanceof Response) { diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 86d2c961..db9fe304 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -252,6 +252,31 @@ export class ApiClient { return await this.request(`/api/sessions/${encodeURIComponent(sessionId)}/files${qs ? `?${qs}` : ''}`) } + async getGeneratedImageBlob(sessionId: string, imageId: string, attempt: number = 0, overrideToken?: string | null): Promise { + const headers = new Headers() + const liveToken = this.getToken ? this.getToken() : null + const authToken = overrideToken !== undefined + ? (overrideToken ?? (liveToken ?? this.token)) + : (liveToken ?? this.token) + if (authToken) { + headers.set('authorization', `Bearer ${authToken}`) + } + const res = await fetch(this.buildUrl(`/api/sessions/${encodeURIComponent(sessionId)}/generated-images/${encodeURIComponent(imageId)}`), { + headers + }) + if (res.status === 401 && attempt === 0 && this.onUnauthorized) { + const refreshed = await this.onUnauthorized() + if (refreshed) { + this.token = refreshed + return await this.getGeneratedImageBlob(sessionId, imageId, attempt + 1, refreshed) + } + } + if (!res.ok) { + throw new ApiError(`HTTP ${res.status}`, res.status, undefined, await res.text().catch(() => undefined)) + } + return await res.blob() + } + async readSessionFile(sessionId: string, path: string): Promise { const params = new URLSearchParams() params.set('path', path) diff --git a/web/src/chat/normalizeAgent.ts b/web/src/chat/normalizeAgent.ts index f7fab5ba..78cd6eea 100644 --- a/web/src/chat/normalizeAgent.ts +++ b/web/src/chat/normalizeAgent.ts @@ -480,6 +480,28 @@ export function normalizeAgentRecord( } } + if (data.type === 'generated-image') { + const imageId = asString(data.imageId ?? data.image_id) + if (!imageId) return null + const uuid = asString(data.id) ?? messageId + return { + id: messageId, + localId, + createdAt, + role: 'agent', + isSidechain: false, + content: [{ + type: 'generated-image', + imageId, + fileName: asString(data.fileName ?? data.file_name) ?? 'generated-image', + mimeType: asString(data.mimeType ?? data.mime_type), + uuid, + parentUUID: null + }], + meta + } + } + if (data.type === 'message' && typeof data.message === 'string') { return { id: messageId, diff --git a/web/src/chat/reconcile.ts b/web/src/chat/reconcile.ts index 12517a2e..321475a0 100644 --- a/web/src/chat/reconcile.ts +++ b/web/src/chat/reconcile.ts @@ -4,6 +4,7 @@ import type { AgentReasoningBlock, AgentTextBlock, ChatBlock, + GeneratedImageBlock, CliOutputBlock, ToolCallBlock, ToolPermission, @@ -135,6 +136,15 @@ function areCliOutputBlocksEqual(left: CliOutputBlock, right: CliOutputBlock): b && left.meta === right.meta } +function areGeneratedImageBlocksEqual(left: GeneratedImageBlock, right: GeneratedImageBlock): boolean { + return left.localId === right.localId + && left.createdAt === right.createdAt + && left.imageId === right.imageId + && left.fileName === right.fileName + && left.mimeType === right.mimeType + && left.meta === right.meta +} + function areAgentEventBlocksEqual(left: AgentEventBlock, right: AgentEventBlock): boolean { return left.createdAt === right.createdAt && left.meta === right.meta @@ -213,6 +223,11 @@ function reconcileBlock(block: ChatBlock, prevById: ChatBlocksById): ChatBlock { return areAgentReasoningBlocksEqual(prevBlock, block) ? prevBlock : block } + if (block.kind === 'generated-image') { + const prevBlock = prev as GeneratedImageBlock + return areGeneratedImageBlocksEqual(prevBlock, block) ? prevBlock : block + } + const prevBlock = prev as AgentEventBlock return areAgentEventBlocksEqual(prevBlock, block) ? prevBlock : block } diff --git a/web/src/chat/reducerTimeline.ts b/web/src/chat/reducerTimeline.ts index 6a372b1f..52dc6d2d 100644 --- a/web/src/chat/reducerTimeline.ts +++ b/web/src/chat/reducerTimeline.ts @@ -730,6 +730,21 @@ export function reduceTimeline( continue } + if (c.type === 'generated-image') { + blocks.push({ + kind: 'generated-image', + id: `${msg.id}:${idx}`, + localId: msg.localId, + createdAt: msg.createdAt, + invokedAt: msg.invokedAt, + imageId: c.imageId, + fileName: c.fileName, + mimeType: c.mimeType, + meta: msg.meta + }) + continue + } + if (c.type === 'reasoning') { blocks.push({ kind: 'agent-reasoning', diff --git a/web/src/chat/types.ts b/web/src/chat/types.ts index 35081cc2..1d9d0f7d 100644 --- a/web/src/chat/types.ts +++ b/web/src/chat/types.ts @@ -56,6 +56,15 @@ export type ToolResult = { permissions?: ToolResultPermission } +export type GeneratedImageContent = { + type: 'generated-image' + imageId: string + fileName: string + mimeType: string | null + uuid: string + parentUUID: string | null +} + export type NormalizedAgentContent = | { type: 'text' @@ -71,6 +80,7 @@ export type NormalizedAgentContent = } | ToolUse | ToolResult + | GeneratedImageContent | { type: 'summary'; summary: string } | { type: 'sidechain'; uuid: string; parentUUID: string | null; prompt: string } @@ -175,6 +185,18 @@ export type CliOutputBlock = { meta?: unknown } +export type GeneratedImageBlock = { + kind: 'generated-image' + id: string + localId: string | null + createdAt: number + invokedAt?: number | null + imageId: string + fileName: string + mimeType: string | null + meta?: unknown +} + export type AgentEventBlock = { kind: 'agent-event' id: string @@ -199,4 +221,4 @@ export type ToolCallBlock = { meta?: unknown } -export type ChatBlock = UserTextBlock | AgentTextBlock | AgentReasoningBlock | CliOutputBlock | ToolCallBlock | AgentEventBlock +export type ChatBlock = UserTextBlock | AgentTextBlock | AgentReasoningBlock | CliOutputBlock | ToolCallBlock | GeneratedImageBlock | AgentEventBlock diff --git a/web/src/components/AssistantChat/messages/ToolMessage.tsx b/web/src/components/AssistantChat/messages/ToolMessage.tsx index a58ca3fb..4d7dfdcd 100644 --- a/web/src/components/AssistantChat/messages/ToolMessage.tsx +++ b/web/src/components/AssistantChat/messages/ToolMessage.tsx @@ -1,6 +1,7 @@ +import { useEffect, useState } from 'react' import type { ToolCallMessagePartProps } from '@assistant-ui/react' import type { ChatBlock } from '@/chat/types' -import type { ToolCallBlock } from '@/chat/types' +import type { GeneratedImageBlock, ToolCallBlock } from '@/chat/types' import type { ToolGroupBlock } from '@/chat/toolGroups' import { isObject, safeStringify } from '@hapi/protocol' import { isSubagentToolName } from '@/chat/subagentTool' @@ -37,6 +38,69 @@ function isToolGroupBlock(value: unknown): value is ToolGroupBlock { return true } +function isGeneratedImageBlock(value: unknown): value is GeneratedImageBlock { + if (!isObject(value)) return false + if (value.kind !== 'generated-image') return false + if (typeof value.id !== 'string') return false + if (typeof value.imageId !== 'string') return false + if (typeof value.fileName !== 'string') return false + if (value.mimeType !== null && typeof value.mimeType !== 'string') return false + return true +} + +function GeneratedImageCard(props: { block: GeneratedImageBlock }) { + const ctx = useHappyChatContext() + const [objectUrl, setObjectUrl] = useState(null) + const [error, setError] = useState(null) + + useEffect(() => { + let disposed = false + let nextObjectUrl: string | null = null + + setObjectUrl(null) + setError(null) + void ctx.api.getGeneratedImageBlob(ctx.sessionId, props.block.imageId) + .then((blob) => { + if (disposed) return + nextObjectUrl = URL.createObjectURL(blob) + setObjectUrl(nextObjectUrl) + }) + .catch((err: unknown) => { + if (disposed) return + setError(err instanceof Error ? err.message : 'Failed to load generated image') + }) + + return () => { + disposed = true + if (nextObjectUrl) { + URL.revokeObjectURL(nextObjectUrl) + } + } + }, [ctx.api, ctx.sessionId, props.block.imageId]) + + return ( +
+
+ Generated image ยท {props.block.fileName} +
+ {objectUrl ? ( + {props.block.fileName} + ) : error ? ( +
+ Generated image is unavailable. {error} +
+ ) : ( +
+ )} +
+ ) +} + function isPendingPermissionBlock(block: ChatBlock): boolean { return block.kind === 'tool-call' && block.tool.permission?.status === 'pending' } @@ -105,6 +169,14 @@ function HappyNestedBlockList(props: { ) } + if (block.kind === 'generated-image') { + return ( +
+ +
+ ) + } + if (block.kind === 'agent-event') { const presentation = getEventPresentation(block.event) return ( @@ -185,6 +257,14 @@ export function HappyToolMessage(props: ToolCallMessagePartProps) { ) } + if (isGeneratedImageBlock(artifact)) { + return ( +
+ +
+ ) + } + if (!isToolCallBlock(artifact)) { const argsText = typeof props.argsText === 'string' ? props.argsText.trim() : '' const hasArgsText = argsText.length > 0 diff --git a/web/src/lib/assistant-runtime.ts b/web/src/lib/assistant-runtime.ts index d6a6b055..2ed3eee5 100644 --- a/web/src/lib/assistant-runtime.ts +++ b/web/src/lib/assistant-runtime.ts @@ -63,6 +63,28 @@ function toThreadMessageLike(block: VisibleChatBlock): ThreadMessageLike { } } + if (block.kind === 'generated-image') { + return { + role: 'assistant', + id: `generated-image:${block.id}`, + createdAt: new Date(block.createdAt), + content: [{ + type: 'tool-call', + toolCallId: block.id, + toolName: 'GeneratedImage', + argsText: '', + artifact: block + }], + metadata: { + custom: { + kind: 'tool', + toolCallId: block.id, + invokedAt: block.invokedAt ?? null + } satisfies HappyChatMessageMetadata + } + } + } + if (block.kind === 'agent-reasoning') { const messageId = `assistant:${block.id}` return {