diff --git a/e2e/composer-image-drag.spec.ts b/e2e/composer-image-drag.spec.ts new file mode 100644 index 00000000..459b04a3 --- /dev/null +++ b/e2e/composer-image-drag.spec.ts @@ -0,0 +1,145 @@ +import { expect, test, type Page } from '@playwright/test' + +const fixtureUrl = '/e2e-fixtures/composer-image-drag-fixture.html' +const attachment = (name: string) => `[data-hapi-composer-attachment-id="${name}"]` + +async function loadFixture(page: Page) { + await page.goto(fixtureUrl, { waitUntil: 'domcontentloaded' }) + await page.getByRole('button', { name: 'Load attachments' }).click() + await expect(page.locator(attachment('notes.txt'))).toBeVisible() +} + +async function openImage(page: Page, gallery = 'Composer', zoom = 100) { + await loadFixture(page) + await page.getByRole('region', { name: gallery, exact: true }).getByTitle('Click to zoom').first().click() + const dialog = page.getByRole('dialog') + await expect(dialog).toBeVisible() + for (let scale = 100; scale < zoom; scale += 25) { + await dialog.getByTitle('Zoom in', { exact: true }).click() + } + return dialog +} + +test.describe('touch image gestures', () => { + test.use({ viewport: { width: 390, height: 844 }, hasTouch: true }) + + test('tap opens the composer image and backdrop tap closes it', async ({ page }) => { + await loadFixture(page) + const trigger = page.locator(attachment('first.png')).getByTitle('Click to zoom') + await expect(page.locator(attachment('notes.txt'))).toBeVisible() + await trigger.tap() + const dialog = page.getByRole('dialog') + await expect(dialog).toBeVisible() + await dialog.locator('img').tap() + await expect(dialog).toBeVisible() + await dialog.locator('div.cursor-grab').tap({ position: { x: 5, y: 5 } }) + await expect(dialog).toBeHidden() + }) + + test('pinch and pan retain their scale and stop on release or cancellation', async ({ page, browserName }) => { + test.skip(browserName !== 'chromium', 'Native multi-touch injection uses Chromium CDP.') + const dialog = await openImage(page) + const image = dialog.locator('img') + const box = await image.boundingBox() + const x = box!.x + box!.width / 2 + const y = box!.y + box!.height / 2 + const cdp = await page.context().newCDPSession(page) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [{ x: x - 40, y, id: 1 }, { x: x + 40, y, id: 2 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchMove', touchPoints: [{ x: x - 70, y, id: 1 }, { x: x + 70, y, id: 2 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] }) + await expect.poll(async () => Number((await dialog.getByTitle('Reset zoom').textContent())!.replace('%', ''))).toBeGreaterThan(100) + const pinched = await image.getAttribute('style') + await cdp.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [{ x, y, id: 1 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchMove', touchPoints: [{ x: x + 30, y: y + 25, id: 1 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchEnd', touchPoints: [] }) + await expect(image).not.toHaveAttribute('style', pinched!) + const panned = await image.getAttribute('style') + await page.mouse.move(x + 60, y + 60) + await expect(image).toHaveAttribute('style', panned!) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchStart', touchPoints: [{ x, y, id: 1 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchMove', touchPoints: [{ x: x + 15, y: y + 15, id: 1 }] }) + await cdp.send('Input.dispatchTouchEvent', { type: 'touchCancel', touchPoints: [] }) + const cancelled = await image.getAttribute('style') + await page.mouse.move(x + 80, y + 80) + await expect(image).toHaveAttribute('style', cancelled!) + await cdp.detach() + }) +}) + +for (const width of [1280, 390]) { + test.describe(`${width}px image gestures`, () => { + test.use({ viewport: { width, height: 900 } }) + + for (const gallery of ['Composer', 'Sent']) { + for (const zoom of [100, 250]) { + test(`${gallery} ${zoom}% releases after a click and after a drag`, async ({ page }) => { + const dialog = await openImage(page, gallery, zoom) + const image = dialog.locator('img') + const box = await image.boundingBox() + const x = box!.x + box!.width / 2 + const y = box!.y + box!.height / 2 + await page.mouse.click(x, y) + const afterRelease = await image.getAttribute('style') + await page.mouse.move(x + 45, y + 25, { steps: 4 }) + await expect(image).toHaveAttribute('style', afterRelease!) + await page.mouse.click(x + 45, y + 25) + await page.mouse.move(x + 60, y + 35, { steps: 3 }) + await expect(image).toHaveAttribute('style', afterRelease!) + + await page.mouse.move(x, y) + await page.mouse.down() + await page.mouse.move(x + 35, y + 20, { steps: 4 }) + await expect(dialog).toBeVisible() + await expect(image).not.toHaveAttribute('style', afterRelease!) + await page.mouse.up() + const afterDrag = await image.getAttribute('style') + await page.mouse.move(x + 75, y + 45, { steps: 4 }) + await expect(image).toHaveAttribute('style', afterDrag!) + await expect(dialog.getByTitle('Reset zoom')).toHaveText(`${zoom}%`) + }) + } + } + + test('retains gallery, zoom, reset, backdrop, close, and Escape controls', async ({ page }) => { + const dialog = await openImage(page) + await dialog.getByRole('button', { name: 'Next image' }).click() + await expect(dialog).toHaveAttribute('aria-label', 'second.png') + await dialog.getByTitle('Zoom in', { exact: true }).click() + await expect(dialog.getByTitle('Reset zoom')).toHaveText('125%') + await dialog.getByTitle('Reset zoom').click() + await expect(dialog.getByTitle('Reset zoom')).toHaveText('100%') + await page.keyboard.press('ArrowLeft') + await expect(dialog).toHaveAttribute('aria-label', 'first.png') + await dialog.getByTitle('Close', { exact: true }).click() + await expect(dialog).toBeHidden() + await page.locator(attachment('first.png')).getByTitle('Click to zoom').click() + await page.keyboard.press('Escape') + await expect(dialog).toBeHidden() + await page.locator(attachment('first.png')).getByTitle('Click to zoom').click() + await dialog.locator('div.cursor-grab').click({ position: { x: 5, y: 5 } }) + await expect(dialog).toBeHidden() + }) + + test('retains attachment drag and keyboard ordering and removal', async ({ page }) => { + await loadFixture(page) + const first = page.locator(attachment('first.png')) + await expect(page.locator(attachment('notes.txt'))).toBeVisible() + await first.getByRole('button', { name: 'Reorder attachment first.png' }).press('ArrowRight') + const items = page.locator('[data-hapi-composer-attachment-id]') + await expect(items.first()).toHaveAttribute('data-hapi-composer-attachment-id', 'second.png') + const handle = await first.getByRole('button', { name: 'Reorder attachment first.png' }).boundingBox() + const target = await page.locator(attachment('second.png')).boundingBox() + await page.mouse.move(handle!.x + 10, handle!.y + 10) + await page.mouse.down() + await page.mouse.move(target!.x + 3, target!.y + 30, { steps: 6 }) + await page.mouse.up() + await expect(items.first()).toHaveAttribute('data-hapi-composer-attachment-id', 'first.png') + await expect(page.getByRole('dialog')).toBeHidden() + await page.locator(attachment('notes.txt')).getByRole('button', { name: 'Reorder attachment notes.txt' }).press('Home') + await expect(items.first()).toHaveAttribute('data-hapi-composer-attachment-id', 'notes.txt') + await first.hover() + await first.getByRole('button', { name: 'Remove attachment' }).click() + await expect(first).toHaveCount(0) + }) + }) +} diff --git a/playwright.image-drag.config.ts b/playwright.image-drag.config.ts new file mode 100644 index 00000000..43efcdec --- /dev/null +++ b/playwright.image-drag.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from '@playwright/test' +import base from './playwright.config' + +export default defineConfig({ + ...base, + testMatch: '**/composer-image-drag.spec.ts', + timeout: 60_000, + projects: [ + ...base.projects!, + { name: 'firefox', use: { browserName: 'firefox' } }, + { name: 'webkit', use: { browserName: 'webkit' } }, + ], +}) diff --git a/web/e2e-fixtures/composer-image-drag-fixture.html b/web/e2e-fixtures/composer-image-drag-fixture.html new file mode 100644 index 00000000..951bc1ff --- /dev/null +++ b/web/e2e-fixtures/composer-image-drag-fixture.html @@ -0,0 +1,12 @@ + + + + + + Composer image drag regression + + +
+ + + diff --git a/web/e2e-fixtures/composer-image-drag-fixture.tsx b/web/e2e-fixtures/composer-image-drag-fixture.tsx new file mode 100644 index 00000000..33d34c70 --- /dev/null +++ b/web/e2e-fixtures/composer-image-drag-fixture.tsx @@ -0,0 +1,84 @@ +import { useCallback, useState } from 'react' +import ReactDOM from 'react-dom/client' +import { + AssistantRuntimeProvider, ComposerPrimitive, useAuiState, useLocalRuntime, + type AttachmentAdapter, type ChatModelAdapter, +} from '@assistant-ui/react' +import '../src/index.css' +import { SortableComposerAttachments } from '../src/components/AssistantChat/SortableComposerAttachments' +import { MessageAttachments } from '../src/components/AssistantChat/messages/MessageAttachments' +import { moveAttachmentId, reconcileAttachmentOrder, type AttachmentDropPosition } from '../src/lib/attachmentOrder' + +const svg = '' +const previewUrl = `data:image/svg+xml,${encodeURIComponent(svg)}` +const model: ChatModelAdapter = { async *run() {} } +const adapter: AttachmentAdapter = { + accept: 'image/*,text/plain', + async add({ file }) { + return { + id: file.name, name: file.name, type: 'file', contentType: file.type, file, + status: { type: 'requires-action', reason: 'composer-send' }, + previewUrl: file.type.startsWith('image/') ? previewUrl : undefined, + } + }, + async remove() {}, + async send(attachment) { + return { ...attachment, status: { type: 'complete' }, content: [{ type: 'text', text: attachment.name }] } + }, +} + +function Composer() { + const attachments = useAuiState((s) => s.composer.attachments) + const [order, setOrder] = useState([]) + const ids = reconcileAttachmentOrder(order, attachments.map((attachment) => attachment.id)) + const onReorder = useCallback((active: string, target: string, position: AttachmentDropPosition) => { + setOrder((previous) => moveAttachmentId( + reconcileAttachmentOrder(previous, attachments.map((attachment) => attachment.id)), + active, target, position, + )) + }, [attachments]) + return ( +
+
+ +
+ +
+ ) +} + +function App() { + const runtime = useLocalRuntime(model, { adapters: { attachments: adapter } }) + const [loaded, setLoaded] = useState(false) + const [loading, setLoading] = useState(false) + const loadAttachments = async () => { + setLoading(true) + try { + for (const name of ['first.png', 'second.png', 'notes.txt']) { + await runtime.thread.composer.addAttachment(new File(['fixture'], name, { + type: name.endsWith('.png') ? 'image/png' : 'text/plain', + })) + } + setLoaded(true) + } finally { + setLoading(false) + } + } + return ( + +
+ +
+ +
+ +
+ + ) +} + +ReactDOM.createRoot(document.getElementById('root')!).render() diff --git a/web/src/components/ImagePreview.test.tsx b/web/src/components/ImagePreview.test.tsx index 2cdb4f2a..127a848f 100644 --- a/web/src/components/ImagePreview.test.tsx +++ b/web/src/components/ImagePreview.test.tsx @@ -12,6 +12,52 @@ function renderGallery() { } describe('ImagePreview gallery navigation', () => { + it.each(['mouse', 'touch', 'pen'])('keeps %s preview gestures out of the surrounding attachment drag handler', (pointerType) => { + const onAttachmentPointerDown = vi.fn() + render( +
+ +
+ ) + fireEvent.click(screen.getByTitle('Click to zoom')) + const dialog = screen.getByRole('dialog') + const image = within(dialog).getByRole('img', { name: 'Photo' }) + const canvas = image.parentElement! + const setPointerCapture = vi.fn() + Object.defineProperty(canvas, 'setPointerCapture', { value: setPointerCapture, configurable: true }) + const pointer = (type: string, x: number, y: number, buttons: number) => { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperties(event, { + pointerId: { value: 1 }, pointerType: { value: pointerType }, + button: { value: 0 }, buttons: { value: buttons }, + clientX: { value: x }, clientY: { value: y }, + }) + fireEvent(image, event) + } + + pointer('pointerdown', 100, 100, 1) + expect(setPointerCapture).toHaveBeenCalledWith(1) + expect(onAttachmentPointerDown).not.toHaveBeenCalled() + pointer('pointermove', 140, 120, 1) + expect(image.style.transform).toContain('40px') + pointer('pointerup', 140, 120, 0) + const releasedTransform = image.style.transform + pointer('pointermove', 180, 160, 0) + expect(image.style.transform).toBe(releasedTransform) + + pointer('pointerdown', 180, 160, 1) + pointer('pointermove', 200, 180, 1) + pointer('pointercancel', 200, 180, 0) + const cancelledTransform = image.style.transform + pointer('pointermove', 240, 220, 0) + expect(image.style.transform).toBe(cancelledTransform) + + fireEvent.pointerDown(within(dialog).getByTitle('Zoom in')) + expect(onAttachmentPointerDown).not.toHaveBeenCalled() + fireEvent.click(within(dialog).getByTitle('Zoom in')) + expect(within(dialog).getByTitle('Reset zoom')).toHaveTextContent('125%') + }) + it('navigates between rendered image previews with toolbar buttons', () => { renderGallery() diff --git a/web/src/components/ImagePreview.tsx b/web/src/components/ImagePreview.tsx index 09693e40..0b580ce1 100644 --- a/web/src/components/ImagePreview.tsx +++ b/web/src/components/ImagePreview.tsx @@ -302,6 +302,8 @@ export function ImagePreview(props: { role="dialog" aria-modal="true" aria-label={activePreview.label} + // Keep preview gestures from starting an ancestor's attachment drag. + onPointerDown={stopEvent} >
{activePreview.fileName}