From 7fe397c595cb93a4e86946b9e124312468e99d5a Mon Sep 17 00:00:00 2001 From: wusumac <736139669@qq.com> Date: Fri, 23 Oct 2026 11:43:24 +0800 Subject: [PATCH] feat(web,cli): load workspace markdown images from the session machine Local Markdown image paths now resolve through the session file API and render from a data URL, with loading/failure/retry states and a portal-based viewer. Out-of-workspace paths show a resend hint; the display_image prompt tells agents not to emit local-path Markdown images. via [HAPI](https://hapi.run) Co-Authored-By: HAPI Co-Authored-By: Claude Code --- cli/src/modules/common/displayImagePrompt.ts | 6 + web/src/components/ImagePreview.tsx | 8 +- .../assistant-ui/markdown-image.test.tsx | 149 ++++++++++++++++++ .../assistant-ui/markdown-image.tsx | 91 +++++++++++ .../components/assistant-ui/markdown-text.tsx | 20 +-- web/src/lib/locales/en.ts | 3 + web/src/lib/locales/zh-CN.ts | 3 + 7 files changed, 260 insertions(+), 20 deletions(-) create mode 100644 web/src/components/assistant-ui/markdown-image.test.tsx create mode 100644 web/src/components/assistant-ui/markdown-image.tsx diff --git a/cli/src/modules/common/displayImagePrompt.ts b/cli/src/modules/common/displayImagePrompt.ts index 48dfbf09..5a2ab939 100644 --- a/cli/src/modules/common/displayImagePrompt.ts +++ b/cli/src/modules/common/displayImagePrompt.ts @@ -1,21 +1,26 @@ import { trimIdent } from '@/utils/trimIdent'; +const LOCAL_IMAGE_MARKDOWN_HINT = 'Local filesystem paths in Markdown images (for example ![preview](/Users/name/image.png), ![preview](/tmp/image.png), or ![preview](./image.png)) do not send the image to HAPI and can appear broken on the user’s phone. Send local images with the display tool above; only use Markdown images for reachable HTTP(S) image URLs. If the display tool fails, report the failure instead of inserting a local-path Markdown image.'; + /** * Shared display_image MCP tool hints — one export per tool naming convention. * Inject into flavor system prompts and first-prompt bridge instructions. */ export const DISPLAY_IMAGE_PROMPT_CLAUDE = trimIdent(` When you create or find a local image file that the human user should see, call the tool "mcp__hapi__display_image" with the absolute filesystem path so HAPI can show it inline. + ${LOCAL_IMAGE_MARKDOWN_HINT} This tool sends the image to the human user for inline display in HAPI. It is not an image-reading or image-understanding tool: it does not provide image input to the model and cannot be used to read, inspect, or analyze image contents. `); export const DISPLAY_IMAGE_PROMPT_CODEX = trimIdent(` When you create or find a local image file that the human user should see, call functions.hapi__display_image with the absolute filesystem path. If that exact tool name is unavailable, use an equivalent alias such as hapi__display_image, mcp__hapi__display_image, or hapi_display_image. + ${LOCAL_IMAGE_MARKDOWN_HINT} This tool sends the image to the human user for inline display in HAPI. It is not an image-reading or image-understanding tool: it does not provide image input to the model and cannot be used to read, inspect, or analyze image contents. `); export const DISPLAY_IMAGE_PROMPT_HAPI_MCP = trimIdent(` When you create or find a local image file that the human user should see, call the tool "hapi_display_image" with the absolute filesystem path so HAPI can show it inline. If that exact tool name is unavailable, use an equivalent alias such as display_image or mcp__hapi__display_image. + ${LOCAL_IMAGE_MARKDOWN_HINT} This tool sends the image to the human user for inline display in HAPI. It is not an image-reading or image-understanding tool: it does not provide image input to the model and cannot be used to read, inspect, or analyze image contents. `); @@ -33,6 +38,7 @@ export const DISPLAY_VIDEO_PROMPT_HAPI_MCP = trimIdent(` export const DISPLAY_IMAGE_PROMPT_CURSOR = trimIdent(` When you create or find a local image file that the human user should see, call the tool "display_image" with the absolute filesystem path so HAPI can show it inline. + ${LOCAL_IMAGE_MARKDOWN_HINT} This tool sends the image to the human user for inline display in HAPI. It is not an image-reading or image-understanding tool: it does not provide image input to the model and cannot be used to read, inspect, or analyze image contents. `); diff --git a/web/src/components/ImagePreview.tsx b/web/src/components/ImagePreview.tsx index 1edfeb7d..90a52fdb 100644 --- a/web/src/components/ImagePreview.tsx +++ b/web/src/components/ImagePreview.tsx @@ -1,4 +1,5 @@ import { useCallback, useEffect, useRef, useState, type CSSProperties, type MouseEvent, type PointerEvent, type ReactNode, type SyntheticEvent, type WheelEvent } from 'react' +import { createPortal } from 'react-dom' import { CloseIcon } from '@/components/icons' const MIN_IMAGE_SCALE = 0.25 @@ -38,6 +39,7 @@ export function ImagePreview(props: { buttonClassName?: string imageClassName?: string imageStyle?: CSSProperties + onImageError?: () => void caption?: ReactNode galleryId?: string onTriggerPointerDown?: (event: PointerEvent) => void @@ -292,13 +294,14 @@ export function ImagePreview(props: { alt={props.label} className={props.imageClassName ?? 'max-h-[calc(100vh-14rem)] max-w-full object-contain transition-transform group-hover:scale-[1.01]'} style={props.imageStyle} + onError={props.onImageError} draggable={false} /> {props.caption} {props.fileName} - {viewerOpen ? ( + {viewerOpen ? createPortal(
- + , + document.body ) : null} ) diff --git a/web/src/components/assistant-ui/markdown-image.test.tsx b/web/src/components/assistant-ui/markdown-image.test.tsx new file mode 100644 index 00000000..b585fabb --- /dev/null +++ b/web/src/components/assistant-ui/markdown-image.test.tsx @@ -0,0 +1,149 @@ +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ApiClient } from '@/api/client' +import { HappyChatProvider, type HappyChatContextValue } from '@/components/AssistantChat/context' +import { MarkdownRenderer } from '@/components/MarkdownRenderer' +import { I18nProvider } from '@/lib/i18n-context' + +function renderMarkdown(content: string, options: { + workspace?: string + readSessionFile?: ApiClient['readSessionFile'] + standalone?: boolean +} = {}) { + const readSessionFile = vi.fn(options.readSessionFile ?? (async () => ({ success: true, content: 'aW1hZ2U=' }))) + const chat: HappyChatContextValue = { + api: { readSessionFile } as unknown as ApiClient, + sessionId: 'session-1', + metadata: { path: options.workspace ?? '/Users/ada/project', host: 'local' }, + terminalToolDisplayMode: 'compact', + showSessionSummaryInChat: false, + disabled: false, + onRefresh: () => {}, + hasMoreMessages: false, + isSyncingTail: false, + isLoadingMoreMessages: false, + loadOlderMessagesPreservingScroll: async () => 'loaded', + } + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }) + const tree = (text: string) => ( + + + + + + + + ) + const view = render(tree(content)) + return { ...view, readSessionFile, queryClient, rerenderMarkdown: (text: string) => view.rerender(tree(text)) } +} + +beforeEach(() => { + localStorage.clear() +}) + +describe('Markdown message images', () => { + it.each([false, true])('loads workspace images from the session machine (standalone=%s)', async (standalone) => { + const { readSessionFile } = renderMarkdown('![Screenshot](/Users/ada/project/output/screen%20one.jpg)', { standalone }) + + expect(screen.queryByRole('img')).not.toBeInTheDocument() + expect(await screen.findByRole('img', { name: 'Screenshot' })).toHaveAttribute('src', 'data:image/jpeg;base64,aW1hZ2U=') + expect(readSessionFile).toHaveBeenCalledWith('session-1', '/Users/ada/project/output/screen one.jpg') + + fireEvent.click(screen.getByRole('button', { name: /Screenshot/ })) + expect(screen.getByRole('dialog', { name: 'Screenshot' })).toBeInTheDocument() + }) + + it('loads relative image paths and caches duplicates within the same session', async () => { + const { readSessionFile } = renderMarkdown('![First](./output/screen.png)\n\n![Second](./output/screen.png)') + + expect(await screen.findAllByRole('img')).toHaveLength(2) + expect(readSessionFile).toHaveBeenCalledTimes(1) + expect(readSessionFile).toHaveBeenCalledWith('session-1', './output/screen.png') + }) + + it('preserves Windows paths through Markdown URI encoding', async () => { + const { readSessionFile } = renderMarkdown('![Screenshot](C:/Users/ada/project/screen%20one.png)', { + workspace: 'C:\\Users\\ada\\project', + }) + + expect(await screen.findByRole('img', { name: 'Screenshot' })).toHaveAttribute('src', 'data:image/png;base64,aW1hZ2U=') + expect(readSessionFile).toHaveBeenCalledWith('session-1', 'C:/Users/ada/project/screen one.png') + }) + + it.each([ + '/Users/ada/.local/share/submissions/oauth.jpg', + '/Users/ada/project/../private.png', + '/Users/ada/project-other/image.png', + ])('shows a resend hint for local files outside the workspace: %s', (path) => { + localStorage.setItem('hapi-lang', 'zh-CN') + const { container, readSessionFile } = renderMarkdown(`![授权页](${path})`) + + expect(screen.getByRole('status')).toHaveTextContent('授权页') + expect(screen.getByRole('status')).toHaveTextContent('请让智能体把它作为附件重新发送') + expect(container.querySelector('img')).toBeNull() + expect(container.querySelector('[data-image-preview-trigger]')).toBeNull() + expect(readSessionFile).not.toHaveBeenCalled() + }) + + it('retries a failed session file read', async () => { + let attempts = 0 + renderMarkdown('![Screenshot](output/screen.png)', { + readSessionFile: async () => ++attempts === 1 + ? { success: false, error: 'Session offline' } + : { success: true, content: 'aW1hZ2U=' }, + }) + + expect(await screen.findByRole('button', { name: 'Retry' })).toBeInTheDocument() + expect(screen.queryByRole('img')).not.toBeInTheDocument() + fireEvent.click(screen.getByRole('button', { name: 'Retry' })) + expect(await screen.findByRole('img', { name: 'Screenshot' })).toBeInTheDocument() + expect(attempts).toBe(2) + }) + + it('keeps the open image viewer stable while the cached file is revalidated', async () => { + let finishRead: () => void = () => {} + const pendingRead = new Promise((resolve) => { finishRead = resolve }) + let attempts = 0 + const { queryClient } = renderMarkdown('![Screenshot](output/screen.png)', { + readSessionFile: async () => { + if (++attempts > 1) await pendingRead + return { success: true, content: 'aW1hZ2U=' } + }, + }) + fireEvent.click(await screen.findByRole('button', { name: /Screenshot/ })) + let refresh: Promise + act(() => { refresh = queryClient.refetchQueries() }) + await waitFor(() => expect(attempts).toBe(2)) + expect(screen.getByRole('dialog', { name: 'Screenshot' })).toBeInTheDocument() + await act(async () => { + finishRead() + await refresh + }) + expect(screen.getByRole('dialog', { name: 'Screenshot' })).toBeInTheDocument() + }) + + it('keeps remote images direct, replaces load failures with a compact retry, and resets on source changes', async () => { + const { readSessionFile, rerenderMarkdown } = renderMarkdown('![Remote](https://example.com/first.png)') + expect(screen.getByRole('img')).toHaveAttribute('src', 'https://example.com/first.png') + expect(readSessionFile).not.toHaveBeenCalled() + + fireEvent.error(screen.getByRole('img')) + expect(screen.queryByRole('img')).not.toBeInTheDocument() + expect(screen.getByRole('status')).toHaveTextContent('Image could not be loaded.') + fireEvent.click(screen.getByRole('button', { name: 'Retry' })) + expect(screen.getByRole('img')).toHaveAttribute('src', 'https://example.com/first.png') + + fireEvent.error(screen.getByRole('img')) + rerenderMarkdown('![Remote](https://example.com/second.png)') + await waitFor(() => expect(screen.getByRole('img')).toHaveAttribute('src', 'https://example.com/second.png')) + }) + + it('does not issue image requests for stripped unsafe URLs', () => { + const { container, readSessionFile } = renderMarkdown('![Blocked](file:///Users/ada/private.png)') + expect(container.querySelector('img')).toBeNull() + expect(screen.getByRole('status')).toHaveTextContent('Blocked') + expect(readSessionFile).not.toHaveBeenCalled() + }) +}) diff --git a/web/src/components/assistant-ui/markdown-image.tsx b/web/src/components/assistant-ui/markdown-image.tsx new file mode 100644 index 00000000..d286fee1 --- /dev/null +++ b/web/src/components/assistant-ui/markdown-image.tsx @@ -0,0 +1,91 @@ +import { useState, type ComponentPropsWithoutRef } from 'react' +import { useQuery } from '@tanstack/react-query' +import type { ApiClient } from '@/api/client' +import { ImagePreview } from '@/components/ImagePreview' +import { useOptionalHappyChatContext } from '@/components/AssistantChat/context' +import { classifyNoSchemeHref } from '@/lib/markdown-href-policy' +import { resolveImageMimeType } from '@/lib/file-preview' +import { queryKeys } from '@/lib/query-keys' +import { useTranslation } from '@/lib/use-translation' +import { cn } from '@/lib/utils' + +function ImageStatus(props: { label: string; message: string; onRetry?: () => void }) { + const { t } = useTranslation() + return ( + + + {props.label} + {props.message} + + {props.onRetry ? ( + + ) : null} + + ) +} + +function Preview(props: { src: string; label: string; className?: string; onRetry?: () => void }) { + const { t } = useTranslation() + const [failedSrc, setFailedSrc] = useState(null) + if (failedSrc === props.src) { + return { + setFailedSrc(null) + props.onRetry?.() + }} /> + } + return ( + setFailedSrc(props.src)} + buttonClassName="group my-3 flex max-h-[32rem] min-h-[10rem] w-full items-center justify-center overflow-hidden rounded-xl bg-[var(--app-subtle-bg)] p-2 text-left" + imageClassName={cn('aui-md-img max-h-[30rem] max-w-full object-contain transition-transform group-hover:scale-[1.01]', props.className)} + /> + ) +} + +function SessionImage(props: { api: ApiClient; sessionId: string; path: string; mimeType: string; label: string; className?: string }) { + const { t } = useTranslation() + const query = useQuery({ + queryKey: queryKeys.sessionFile(props.sessionId, props.path), + queryFn: () => props.api.readSessionFile(props.sessionId, props.path), + staleTime: 60_000, + retry: false, + }) + const retry = () => { void query.refetch() } + if (query.isPending || (query.isFetching && !query.data?.content)) { + return + } + if (!query.data?.success || !query.data.content) { + return + } + const src = `data:${props.mimeType};base64,${query.data.content}` + return +} + +/** Local Markdown paths belong to the agent machine, never to the web origin. */ +export function MarkdownImage(props: ComponentPropsWithoutRef<'img'>) { + const chat = useOptionalHappyChatContext() + const { t } = useTranslation() + const src = typeof props.src === 'string' ? props.src : '' + const label = props.alt || t('media.displayed.image') + if (!src) return + + const localPath = /^[A-Za-z]:(?:[\\/]|%5c|%2f)/i.test(src) + || !/^(?:[a-z][a-z\d+.-]*:|\/\/)/i.test(src) + if (chat && localPath) { + const decision = classifyNoSchemeHref(src, { workspacePath: chat.metadata?.path }) + const mimeType = decision.action === 'file' ? resolveImageMimeType(decision.path) : null + if (decision.action !== 'file' || !mimeType) { + // Outside-workspace media must be sent through the permission-gated + // display_image tool; Markdown text cannot grant filesystem access. + return + } + return + } + + return +} diff --git a/web/src/components/assistant-ui/markdown-text.tsx b/web/src/components/assistant-ui/markdown-text.tsx index 5a5b42fe..f9adb165 100644 --- a/web/src/components/assistant-ui/markdown-text.tsx +++ b/web/src/components/assistant-ui/markdown-text.tsx @@ -32,7 +32,7 @@ import { classifyNoSchemeHref } from '@/lib/markdown-href-policy' import { remarkSessionPathLinks } from '@/lib/remark-session-path-links' import { buildSessionReferencePath, parseSessionPathHref } from '@/lib/sessionReference' import { UriConfirmDialog } from '@/components/UriConfirmDialog' -import { ImagePreview } from '@/components/ImagePreview' +import { MarkdownImage } from '@/components/assistant-ui/markdown-image' import { isPreviewableFilePath } from '@/lib/file-preview' import { useOptionalFilePreview } from '@/lib/file-preview-context' @@ -851,22 +851,6 @@ function Em(props: ComponentPropsWithoutRef<'em'>) { return } -function Image(props: ComponentPropsWithoutRef<'img'>) { - const src = typeof props.src === 'string' ? props.src : '' - const label = typeof props.alt === 'string' && props.alt ? props.alt : 'Image preview' - if (!src) return - - return ( - - ) -} - type DefaultComponentsMap = { [key: string]: ComponentType code: ComponentType @@ -901,7 +885,7 @@ export const defaultComponents: DefaultComponentsMap = memoizeMarkdownComponents tr: Tr, th: Th, td: Td, - img: Image, + img: MarkdownImage, } as const) as unknown as DefaultComponentsMap export function MarkdownText({ smooth }: { smooth?: boolean } = {}) { diff --git a/web/src/lib/locales/en.ts b/web/src/lib/locales/en.ts index d04f8253..7b3cffda 100644 --- a/web/src/lib/locales/en.ts +++ b/web/src/lib/locales/en.ts @@ -585,6 +585,9 @@ export default { 'media.displayed.file': 'Displayed file', 'media.displayed.header': '{label}: {fileName}', 'media.displayed.unavailable': '{label} is unavailable. {error}', + 'media.image.loading': 'Loading image…', + 'media.image.loadFailed': 'Image could not be loaded.', + 'media.image.localUnavailable': 'This image is on the agent’s computer. Ask the agent to resend it as an attachment.', // Tool card 'tool.askQuestion': 'Other', diff --git a/web/src/lib/locales/zh-CN.ts b/web/src/lib/locales/zh-CN.ts index 4bf62824..adef51ae 100644 --- a/web/src/lib/locales/zh-CN.ts +++ b/web/src/lib/locales/zh-CN.ts @@ -584,6 +584,9 @@ export default { 'media.displayed.file': '展示文件', 'media.displayed.header': '{label}:{fileName}', 'media.displayed.unavailable': '{label}不可用:{error}', + 'media.image.loading': '正在加载图片…', + 'media.image.loadFailed': '图片加载失败。', + 'media.image.localUnavailable': '这张图片在智能体的电脑上,请让智能体把它作为附件重新发送。', // Tool card 'tool.askQuestion': '其他',