mirror of
https://github.com/wu736139669/hapi.git
synced 2026-11-18 03:27:08 +00:00
* feat(cli): cross-flavor inline image display via MCP and ACP Share display_image prompt across MCP-bridge flavors (Cursor, Gemini, Kimi, Codex, Claude, OpenCode), auto-approve the tool in buildHapiMcpBridge, handle ACP image content blocks, and harden generated-image registration with content sniffing. Closes tiann/hapi#956 Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): render generated-image cards reliably in chat Keep object URLs stable across refetch, upscale tiny inline images, fetch generated-image bytes with cache no-store (avoid empty 304 bodies), and load hapiMcpUrl from per-session API in hapi-display-image tooling. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(cli+web): display_video MCP for inline mp4/webm (#956) Add display_video alongside display_image, video MIME sniffing with avif guard, web GeneratedImageCard video player, and hapi-display-image auto-routing. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(cli+web): cross-flavor display_video parity with images (#956) Share display_video prompts across MCP-bridge flavors, auto-approve the tool, register mp4/webm via path sniffing, render inline video in web on the existing generated-image RPC path, and restore robust media card fetch. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): ACP image ordering and inline media source provenance Flush buffered assistant text before async generated_image emit from ACP image blocks (PR #958 review Major). Add optional source metadata on generated-image wire messages (ingress, flavor, toolCallId, toolName) for MCP, ACP, and Codex tool-result paths. Seeds artifact-event follow-up #966. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli+web): address PR #958 review Majors on media order and stale blobs Queue ACP session updates and await async image registration before later events; clear GeneratedImageCard blob state when imageId changes. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): await ACP queue after late-drain before turn_complete Straggler session/update during drainLateBuffers can queue async image registration; re-await sessionUpdateQueue so generated_image is not emitted after turn_complete. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(scripts): route AVIF ftyp brands to display_image in helper Match server-side detectImageMimeType so .avif files are not sent to display_video and rejected as unsupported video. Co-authored-by: Cursor <cursoragent@cursor.com> * feat(#956): agent inline-media doctor and discovery fixes - hapi doctor inline-media: probe bridges, print per-session inline commands - Expose hapiMcpUrl on session list summaries (stops false "no MCP" scans) - Helper script: match cursorSessionId prefixes; HAPI_SESSION_ID path-only mode - ACP bridge prompt: shell fallback + HAPI session id vs agent id rule Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): allow immutable cache for generated media blobs Drop cache: no-store on generated-image fetch so browser can reuse hub immutable responses; on 304 re-read via force-cache (#927, PR review). Co-authored-by: Cursor <cursoragent@cursor.com> * fix(#956): Cursor native MCP overlay; drop user-turn bridge prepend Cursor ACP ignores session/new mcpServers. Write .cursor/mcp.json and run agent mcp enable hapi instead. Remove HAPI_MCP_BRIDGE_PROMPT from user turns on ACP remotes; enrich MCP tool descriptions for discovery. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): preserve non-HAPI mcp.json keys on Cursor overlay cleanup Cleanup only removes or restores the hapi MCP entry instead of rewriting the full pre-session snapshot, so concurrent edits to other servers survive. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): handle generated_image in Grok ACP launcher switch Upstream Grok launcher exhaustiveness broke after AgentMessage gained generated_image for cross-flavor inline media. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): rebase fallout for display_video + OpenCode skill lookup Gate display_video in the STDIO bridge, restore OpenCode first-prompt TITLE_INSTRUCTION (skill_lookup), and update tool-list test expectations. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): leave user-owned hapi MCP entry alone on overlay cleanup Only undo mcpServers.hapi when it still matches the exact entry this session installed; concurrent Cursor/user edits of that key survive. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): exact-match auto-approve for display_image and display_video Move media tools off substring name/id hints onto the exact-name set so forged lookalike tools are not approved in default permission mode. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): drop dead Cursor bridge prompt; put guidance in MCP descriptions Cursor must not get a user-turn media prepend (prompt-taint). Remove unused HAPI_MCP_BRIDGE_PROMPT_CURSOR and embed DISPLAY_*_PROMPT_CURSOR in the display_image/display_video MCP tool descriptions instead. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): require user approval for display_image and display_video Those tools read arbitrary local paths into chat; keep them on MCP approval_mode prompt and out of default-mode auto-approve exact names. Co-authored-by: Cursor <cursoragent@cursor.com> * chore: re-trigger Codex PR review after provider 503 Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): ignore URI-only ACP image blocks that read local disk Passive ACP agentMessageChunk handling must not load file:// or bare paths; local media goes through prompt-gated display_image/display_video. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): allowlist MP4 ftyp brands for video sniffing Reject HEIC/HEIF and other non-video ISO-BMFF containers instead of treating every non-AVIF ftyp as video/mp4. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): keep Cursor ACP startup if MCP overlay fails Wrap installCursorMcpOverlay so a malformed project .cursor/mcp.json cannot abort the session; continue without inline media tools. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): fail doctor inline-media when checks fail Exit non-zero whenever required checks fail, even if an active hapiMcpUrl bridge is present. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): expect display_video when change_title is disabled Native ACP title mode still exposes display_image and display_video; update startHappyServer test after rebase onto 0.23.4. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): keep ACP title sync synchronous outside media queue session_info_update title forwarding (#1028) must not wait on the async message-handler queue used for inline media ordering. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): sniff media headers only; advertise OpenCode display_video Read 16 bytes for detectMediaTool instead of the whole file, and include hapi_display_video in OPENCODE_NATIVE_TOOL_INSTRUCTION for remote ACP. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): show Cursor generate_image inline in HAPI chat cursor/generate_image only emitted a tool card; register filePath or base64 imageData into generatedImages and emit generated_image so the web chat card renders (issue #956 / swear01 report). Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): ignore path-only Cursor generate_image reads Path-only filePath registration bypassed permission-gated display_image / display_video MCP tools. Keep base64 imageData only; local paths must go through MCP approval. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): gate inline media base64 length before decode Reject oversized ACP/Cursor base64 payloads by character count so the CLI never allocates past the 25 MB generated-image cap. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): require EBML DocType webm for inline video sniff Bare EBML magic matches Matroska/MKV too; only accept DocType webm. Also restore annotated Playwright cursor in annotatedVideoUseOption. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(scripts): read 128-byte header for WebM DocType sniff detectMediaTool only loaded 16 bytes, so EBML DocType webm was often missing and valid WebM files fell through to display_image. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): compare generated-image source by value in reconcile Wire normalization allocates a fresh source object each pass; reference equality forced media-card recomputation on every reload/SSE refresh. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): inject Cursor MCP enable for overlay unit tests installCursorMcpOverlay always spawned `agent mcp enable`; tests now pass a noop so the suite never shells out to a real Cursor binary. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): shell-quote doctor inline-media helper command Paths and session prefixes with spaces/metacharacters broke the copied snippet; JSON.stringify each interpolated argument. Co-authored-by: Cursor <cursoragent@cursor.com> * test(cli): fix doctor inline-media quote path expectation Repo root from scriptPath is three levels up (cli/), not the parent of cli. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli/web): per-session Cursor MCP overlay id and bound tiny-image scale Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): redact generate_image base64 from logs and fix doctor MCP ids Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): harden inline-media doctor for packaged installs and hub headers Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): lock Cursor mcp.json updates and bound ACP media filenames Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): preserve mcp.json mode and token-scoped overlay locks Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): publish Cursor MCP lock owners via link(2) and treat EPERM as alive Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): fail closed on stale MCP locks; keep concurrent mcp.json top-level keys Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): roll back Cursor MCP overlay when agent mcp enable fails Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): await ACP session queue in suppressUpdatesDuring tests #958 queues handleUpdate for media registration; upstream compact tests assumed sync delivery after restore. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): capture ACP handler at enqueue; keep display_video manual Close two Major review findings on #958: suppress queue leak after restore, and Claude --allowedTools auto-approving local-path video. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: write through symlinked mcp.json; lazy-load inline video Preserve user Cursor MCP symlinks on atomic overlay writes, and require explicit Load video before fetching large generated-video blobs. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): use valid TerminalToolDisplayMode in media card test Co-authored-by: Cursor <cursoragent@cursor.com> * fix(scripts): require unique session prefix in display helper Reject ambiguous prefix matches so images/videos cannot land in the wrong HAPI chat when multiple agent session ids share a prefix. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): always cleanup Cursor MCP overlay on teardown Run overlay cleanup in finally so cancelAll/disconnect failures cannot leave a dead hapi-<sessionId> entry in .cursor/mcp.json. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): handle Copilot generated_image; refuse MCP symlinks Unblock typecheck after Antigravity/Copilot merge, and fail closed when .cursor/mcp.json or .cursor is a project-controlled symlink. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: abort restore deliveryMode; prune dead Cursor MCP overlays Unblock web typecheck after steer merge, and recover orphaned hapi-* mcp.json entries via HAPI_MCP_OVERLAY_PID ownership stamps. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): recover dead-PID Cursor MCP overlay locks Token-matched unlock so a crash mid-lock no longer permanently disables inline media; keep live-owner waits identity-safe. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): serialize Cursor MCP stale-lock recovery Acquire an exclusive recovery lock before token-matched unlink so two recoverers cannot remove a successor's live mcp.json lock. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): fail closed on stale Cursor MCP overlay locks Withdraw racy auto-recovery: pathname check-then-unlink/rename can steal a successor lock. Stale locks throw with an explicit rm hint instead. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(web): drop duplicate deliveryMode on abort restore Merge left both steer and queue; keep queue so retries after abort do not re-bind to a later Pi turn. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): bound inline media reads on open fd Close TOCTOU between pathname size check and readFile for display_image / display_video and registerGeneratedImageFromPath. Also preserve non-PID env edits on Cursor MCP overlay cleanup. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): repair happyMcpStdioBridge test syntax after merge Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): write Cursor MCP overlay to ~/.cursor, not the project Keep ephemeral hapi-<sessionId> bridges out of the checked-out tree so agents cannot git-add a live loopback URL. Tests inject mcpConfigDir. Co-authored-by: Cursor <cursoragent@cursor.com> * fix(cli): point Cursor MCP diagnostics at ~/.cursor/mcp.json Co-authored-by: Cursor <cursoragent@cursor.com> * feat(media): add audio and file display --------- Co-authored-by: HeavyGee <133152184+heavygee@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Debian <heavygee@oos-linux.in.lockhouse>
445 lines
14 KiB
TypeScript
445 lines
14 KiB
TypeScript
import { basename } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { open } from 'node:fs/promises'
|
|
import { randomUUID } from 'node:crypto'
|
|
import { asString, isObject } from '@hapi/protocol'
|
|
|
|
/**
|
|
* Read a regular file through one open descriptor: size is taken from the fd
|
|
* (not a separate pathname `lstat`), then the buffer is allocated to that size
|
|
* only. Rejects if the file grows/shrinks while reading so a TOCTOU swap cannot
|
|
* force an unbounded `readFile` allocation.
|
|
*/
|
|
export async function readBoundedRegularFile(path: string, maxBytes: number): Promise<Buffer> {
|
|
const handle = await open(path, 'r')
|
|
try {
|
|
const info = await handle.stat()
|
|
if (!info.isFile()) {
|
|
throw new Error('Path is not a regular file')
|
|
}
|
|
if (info.size > maxBytes) {
|
|
throw new Error('File is too large to display inline')
|
|
}
|
|
const bytes = Buffer.alloc(info.size)
|
|
for (let offset = 0; offset < bytes.length;) {
|
|
const { bytesRead } = await handle.read(bytes, offset, bytes.length - offset, offset)
|
|
if (bytesRead === 0) {
|
|
throw new Error('File changed while reading')
|
|
}
|
|
offset += bytesRead
|
|
}
|
|
if ((await handle.stat()).size !== info.size) {
|
|
throw new Error('File changed while reading')
|
|
}
|
|
return bytes
|
|
} finally {
|
|
await handle.close()
|
|
}
|
|
}
|
|
|
|
export type GeneratedImageMetadata = {
|
|
id: string
|
|
fileName: string
|
|
content: Buffer
|
|
mimeType: string
|
|
createdAt: number
|
|
}
|
|
|
|
export const MAX_GENERATED_IMAGE_BYTES = 25 * 1024 * 1024
|
|
/** Reject base64 strings that cannot fit under MAX_GENERATED_IMAGE_BYTES once decoded (+ padding). */
|
|
export const MAX_GENERATED_IMAGE_BASE64_CHARS = Math.ceil(MAX_GENERATED_IMAGE_BYTES * 4 / 3) + 4
|
|
const MAX_GENERATED_IMAGE_TOTAL_BYTES = 100 * 1024 * 1024
|
|
const MAX_GENERATED_IMAGE_COUNT = 100
|
|
|
|
/** Decode inline media base64 only after a cheap length gate (avoids huge Buffer allocations). */
|
|
export function decodeGeneratedImageBase64(data: string): Buffer | null {
|
|
if (data.length > MAX_GENERATED_IMAGE_BASE64_CHARS) {
|
|
return null
|
|
}
|
|
const bytes = Buffer.from(data, 'base64')
|
|
if (bytes.byteLength > MAX_GENERATED_IMAGE_BYTES) {
|
|
return null
|
|
}
|
|
return bytes
|
|
}
|
|
|
|
const generatedImages = new Map<string, GeneratedImageMetadata>()
|
|
let generatedImageBytes = 0
|
|
|
|
export function detectImageMimeType(bytes: Uint8Array): string | null {
|
|
if (bytes.length >= 8
|
|
&& bytes[0] === 0x89
|
|
&& bytes[1] === 0x50
|
|
&& bytes[2] === 0x4e
|
|
&& bytes[3] === 0x47
|
|
&& bytes[4] === 0x0d
|
|
&& bytes[5] === 0x0a
|
|
&& bytes[6] === 0x1a
|
|
&& bytes[7] === 0x0a) {
|
|
return 'image/png'
|
|
}
|
|
|
|
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
|
return 'image/jpeg'
|
|
}
|
|
|
|
if (bytes.length >= 6) {
|
|
const header = ascii(bytes, 0, 6)
|
|
if (header === 'GIF87a' || header === 'GIF89a') {
|
|
return 'image/gif'
|
|
}
|
|
}
|
|
|
|
if (bytes.length >= 12 && ascii(bytes, 0, 4) === 'RIFF' && ascii(bytes, 8, 12) === 'WEBP') {
|
|
return 'image/webp'
|
|
}
|
|
|
|
if (bytes.length >= 12
|
|
&& bytes[0] === 0x00
|
|
&& bytes[1] === 0x00
|
|
&& bytes[2] === 0x00
|
|
&& ascii(bytes, 4, 8) === 'ftyp'
|
|
&& (ascii(bytes, 8, 12) === 'avif' || ascii(bytes, 8, 12) === 'avis')) {
|
|
return 'image/avif'
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
const MP4_FTYP_BRANDS = new Set([
|
|
'isom',
|
|
'iso2',
|
|
'iso3',
|
|
'iso4',
|
|
'iso5',
|
|
'iso6',
|
|
'mp41',
|
|
'mp42',
|
|
'mp71',
|
|
'avc1',
|
|
'avc3',
|
|
'hev1',
|
|
'hvc1',
|
|
'mmp4',
|
|
'dash',
|
|
'msnv',
|
|
'ndas',
|
|
'ndsc',
|
|
'ndsh',
|
|
'ndsm',
|
|
'ndsp',
|
|
'ndss',
|
|
'ndxc',
|
|
'ndxh',
|
|
'ndxm',
|
|
'ndxp',
|
|
'ndxs',
|
|
])
|
|
|
|
/** EBML DocType (element 0x4282) from the first header bytes; null if absent/unreadable. */
|
|
function readEbmlDocType(bytes: Uint8Array): string | null {
|
|
const limit = Math.min(bytes.length, 128)
|
|
for (let i = 4; i + 3 < limit; i += 1) {
|
|
if (bytes[i] !== 0x42 || bytes[i + 1] !== 0x82) {
|
|
continue
|
|
}
|
|
const sizeByte = bytes[i + 2]
|
|
// Single-byte VINT: high bit set, length in low 7 bits.
|
|
if ((sizeByte & 0x80) === 0) {
|
|
continue
|
|
}
|
|
const len = sizeByte & 0x7f
|
|
if (len === 0 || i + 3 + len > limit) {
|
|
continue
|
|
}
|
|
return ascii(bytes, i + 3, i + 3 + len)
|
|
}
|
|
return null
|
|
}
|
|
|
|
export function detectVideoMimeType(bytes: Uint8Array): string | null {
|
|
if (bytes.length >= 12 && ascii(bytes, 4, 8) === 'ftyp') {
|
|
const brand = ascii(bytes, 8, 12)
|
|
if (MP4_FTYP_BRANDS.has(brand)) {
|
|
return 'video/mp4'
|
|
}
|
|
return null
|
|
}
|
|
|
|
if (bytes.length >= 4
|
|
&& bytes[0] === 0x1a
|
|
&& bytes[1] === 0x45
|
|
&& bytes[2] === 0xdf
|
|
&& bytes[3] === 0xa3) {
|
|
// EBML is shared by WebM and Matroska/MKV — only accept DocType webm.
|
|
if (readEbmlDocType(bytes) === 'webm') {
|
|
return 'video/webm'
|
|
}
|
|
return null
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
export function detectAudioMimeType(bytes: Uint8Array): string | null {
|
|
if (bytes.length >= 12 && ascii(bytes, 0, 4) === 'RIFF' && ascii(bytes, 8, 12) === 'WAVE') {
|
|
return 'audio/wav'
|
|
}
|
|
|
|
if (bytes.length >= 4 && ascii(bytes, 0, 4) === 'fLaC') {
|
|
return 'audio/flac'
|
|
}
|
|
|
|
if (bytes.length >= 32 && ascii(bytes, 0, 4) === 'OggS') {
|
|
const probe = ascii(bytes, 0, Math.min(bytes.length, 96))
|
|
if (probe.includes('OpusHead') || probe.includes('vorbis') || probe.includes('Speex')) {
|
|
return 'audio/ogg'
|
|
}
|
|
}
|
|
|
|
if (bytes.length >= 3 && ascii(bytes, 0, 3) === 'ID3') {
|
|
return 'audio/mpeg'
|
|
}
|
|
|
|
if (bytes.length >= 3
|
|
&& bytes[0] === 0xff
|
|
&& (bytes[1] & 0xe0) === 0xe0
|
|
&& (bytes[1] & 0x18) !== 0x08
|
|
&& (bytes[1] & 0x06) !== 0
|
|
&& (bytes[2] & 0xf0) !== 0
|
|
&& (bytes[2] & 0xf0) !== 0xf0
|
|
&& (bytes[2] & 0x0c) !== 0x0c) {
|
|
return 'audio/mpeg'
|
|
}
|
|
|
|
if (bytes.length >= 12 && ascii(bytes, 4, 8) === 'ftyp') {
|
|
const brand = ascii(bytes, 8, 12)
|
|
if (brand === 'M4A ' || brand === 'M4B ' || brand === 'M4P ') {
|
|
return 'audio/mp4'
|
|
}
|
|
}
|
|
|
|
return null
|
|
}
|
|
|
|
function detectIsoBmffTrackKind(bytes: Uint8Array): 'audio' | 'video' | null {
|
|
const buffer = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
let hasAudio = false
|
|
let hasVideo = false
|
|
|
|
function visitBoxes(start: number, end: number, parent: 'root' | 'moov' | 'trak' | 'mdia'): boolean {
|
|
let offset = start
|
|
for (; offset + 8 <= end;) {
|
|
const size32 = buffer.readUInt32BE(offset)
|
|
const type = ascii(bytes, offset + 4, offset + 8)
|
|
let headerSize = 8
|
|
let boxSize: number
|
|
if (size32 === 0) {
|
|
boxSize = end - offset
|
|
} else if (size32 === 1) {
|
|
if (offset + 16 > end) return false
|
|
const extendedSize = buffer.readBigUInt64BE(offset + 8)
|
|
if (extendedSize > BigInt(Number.MAX_SAFE_INTEGER)) return false
|
|
headerSize = 16
|
|
boxSize = Number(extendedSize)
|
|
} else {
|
|
boxSize = size32
|
|
}
|
|
if (boxSize < headerSize || boxSize > end - offset) return false
|
|
|
|
const payloadStart = offset + headerSize
|
|
const boxEnd = offset + boxSize
|
|
if (parent === 'root' && type === 'moov') {
|
|
if (!visitBoxes(payloadStart, boxEnd, 'moov')) return false
|
|
} else if (parent === 'moov' && type === 'trak') {
|
|
if (!visitBoxes(payloadStart, boxEnd, 'trak')) return false
|
|
} else if (parent === 'trak' && type === 'mdia') {
|
|
if (!visitBoxes(payloadStart, boxEnd, 'mdia')) return false
|
|
} else if (parent === 'mdia' && type === 'hdlr') {
|
|
if (boxSize < headerSize + 12) return false
|
|
const handlerType = ascii(bytes, payloadStart + 8, payloadStart + 12)
|
|
if (handlerType === 'soun') hasAudio = true
|
|
if (handlerType === 'vide') hasVideo = true
|
|
}
|
|
|
|
offset = boxEnd
|
|
}
|
|
return offset === end
|
|
}
|
|
|
|
if (!visitBoxes(0, bytes.length, 'root')) return null
|
|
if (hasVideo) return 'video'
|
|
if (hasAudio) return 'audio'
|
|
return null
|
|
}
|
|
|
|
export function isInlineMediaMimeType(mimeType: string): boolean {
|
|
return mimeType.startsWith('image/') || mimeType.startsWith('video/') || mimeType.startsWith('audio/')
|
|
}
|
|
|
|
export function detectDisplayMediaMimeType(bytes: Uint8Array): string {
|
|
const imageMimeType = detectImageMimeType(bytes)
|
|
if (imageMimeType) return imageMimeType
|
|
|
|
if (bytes.length >= 12 && ascii(bytes, 4, 8) === 'ftyp') {
|
|
const trackKind = detectIsoBmffTrackKind(bytes)
|
|
if (trackKind === 'video') return 'video/mp4'
|
|
if (trackKind === 'audio') return 'audio/mp4'
|
|
return detectAudioMimeType(bytes) ?? 'application/octet-stream'
|
|
}
|
|
|
|
return detectVideoMimeType(bytes)
|
|
?? detectAudioMimeType(bytes)
|
|
?? 'application/octet-stream'
|
|
}
|
|
|
|
function ascii(bytes: Uint8Array, start: number, end: number): string {
|
|
return String.fromCharCode(...bytes.subarray(start, end))
|
|
}
|
|
|
|
export function registerGeneratedImage(args: { id: string; path: string; mimeType: string; bytes: Uint8Array; fileName?: string | null }): GeneratedImageMetadata {
|
|
const content = Buffer.from(args.bytes)
|
|
if (content.byteLength > MAX_GENERATED_IMAGE_BYTES) {
|
|
throw new Error('File is too large to display inline')
|
|
}
|
|
|
|
if (!isInlineMediaMimeType(args.mimeType) && args.mimeType !== 'application/octet-stream') {
|
|
throw new Error('Unsupported generated media MIME type')
|
|
}
|
|
|
|
const previous = generatedImages.get(args.id)
|
|
if (previous) {
|
|
generatedImageBytes -= previous.content.byteLength
|
|
}
|
|
|
|
const metadata: GeneratedImageMetadata = {
|
|
id: args.id,
|
|
fileName: args.fileName || basename(args.path) || `${args.id}.png`,
|
|
content,
|
|
mimeType: args.mimeType,
|
|
createdAt: Date.now()
|
|
}
|
|
generatedImages.set(args.id, metadata)
|
|
generatedImageBytes += content.byteLength
|
|
|
|
evictOldGeneratedImages()
|
|
|
|
return metadata
|
|
}
|
|
|
|
function evictOldGeneratedImages(): void {
|
|
while (generatedImages.size > MAX_GENERATED_IMAGE_COUNT || generatedImageBytes > MAX_GENERATED_IMAGE_TOTAL_BYTES) {
|
|
const oldestId = generatedImages.keys().next().value
|
|
if (!oldestId) break
|
|
const oldest = generatedImages.get(oldestId)
|
|
if (oldest) {
|
|
generatedImageBytes -= oldest.content.byteLength
|
|
}
|
|
generatedImages.delete(oldestId)
|
|
}
|
|
}
|
|
|
|
export function getGeneratedImage(id: string): GeneratedImageMetadata | null {
|
|
return generatedImages.get(id) ?? null
|
|
}
|
|
|
|
export function clearGeneratedImages(): void {
|
|
generatedImages.clear()
|
|
generatedImageBytes = 0
|
|
}
|
|
|
|
export async function registerGeneratedImageFromPath(args: {
|
|
id?: string
|
|
path: string
|
|
fileName?: string | null
|
|
}): Promise<GeneratedImageMetadata | null> {
|
|
try {
|
|
const bytes = await readBoundedRegularFile(args.path, MAX_GENERATED_IMAGE_BYTES)
|
|
const mimeType = detectImageMimeType(bytes) ?? detectVideoMimeType(bytes)
|
|
if (!mimeType) {
|
|
throw new Error('Unsupported inline media content')
|
|
}
|
|
return registerGeneratedImage({
|
|
id: args.id ?? randomUUID(),
|
|
path: args.path,
|
|
fileName: args.fileName,
|
|
mimeType,
|
|
bytes
|
|
})
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
|
|
function parseAcpImageUri(uri: string): string | null {
|
|
if (uri.startsWith('file://')) {
|
|
try {
|
|
return fileURLToPath(uri)
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
if (/^https?:\/\//i.test(uri)) {
|
|
return null
|
|
}
|
|
return uri
|
|
}
|
|
|
|
/** Bound safe display name from an ACP uri/url — never reuse raw data/signed URLs. */
|
|
export function safeAcpFileName(uri: string | undefined | null): string | null {
|
|
if (!uri) return null
|
|
try {
|
|
if (uri.startsWith('file://')) {
|
|
return basename(fileURLToPath(uri)).slice(0, 255) || null
|
|
}
|
|
if (/^https?:\/\//i.test(uri)) {
|
|
const name = basename(new URL(uri).pathname)
|
|
return name.slice(0, 255) || null
|
|
}
|
|
// Reject scheme-bearing forms (data:, blob:, etc.) — only bare paths.
|
|
if (!uri.includes(':')) {
|
|
return basename(uri).slice(0, 255) || null
|
|
}
|
|
} catch {
|
|
return null
|
|
}
|
|
return null
|
|
}
|
|
|
|
export async function registerGeneratedImageFromAcpBlock(block: unknown): Promise<GeneratedImageMetadata | null> {
|
|
if (!isObject(block) || block.type !== 'image') {
|
|
return null
|
|
}
|
|
|
|
const data = asString(block.data)
|
|
const declaredMimeType = asString(block.mimeType ?? block.mime_type)
|
|
const uri = asString(block.uri ?? block.url)
|
|
|
|
if (data) {
|
|
const bytes = decodeGeneratedImageBase64(data)
|
|
if (!bytes) {
|
|
return null
|
|
}
|
|
const sniffedMimeType = detectImageMimeType(bytes)
|
|
if (!sniffedMimeType) {
|
|
return null
|
|
}
|
|
if (declaredMimeType && declaredMimeType !== sniffedMimeType) {
|
|
return null
|
|
}
|
|
const localPath = uri ? parseAcpImageUri(uri) : null
|
|
const fileName = safeAcpFileName(uri) ?? `generated-${randomUUID()}.png`
|
|
return registerGeneratedImage({
|
|
id: randomUUID(),
|
|
path: localPath ?? fileName,
|
|
fileName,
|
|
mimeType: sniffedMimeType,
|
|
bytes
|
|
})
|
|
}
|
|
|
|
// URI-only ACP image blocks are not permission-gated. Local-path display must
|
|
// go through display_image / display_video / display_media MCP tools (approval_mode: prompt).
|
|
return null
|
|
}
|