diff --git a/ios/Hapi/Features/Chat/Blocks/ChatBlockCard.swift b/ios/Hapi/Features/Chat/Blocks/ChatBlockCard.swift index 0064d05c..626faef0 100644 --- a/ios/Hapi/Features/Chat/Blocks/ChatBlockCard.swift +++ b/ios/Hapi/Features/Chat/Blocks/ChatBlockCard.swift @@ -27,63 +27,13 @@ extension VisibleChatBlock { // MARK: - Media plumbing -/// Per-chat generated-image loading: `GET /generated-images/:imageId` bytes -/// through the authed APIClient (URLCache underneath serves immutable-ETag -/// repeats), plus a small in-memory decode cache so scroll-back does not -/// re-decode. Injected down the block tree via `\.chatMedia`. -@MainActor -final class GeneratedImageLoader { - private let api: APIClient - private let sessionId: String - private let cache = NSCache() - private var inFlight: [String: Task] = [:] - - init(api: APIClient, sessionId: String) { - self.api = api - self.sessionId = sessionId - cache.countLimit = 40 - } - - func image(for imageId: String) async -> UIImage? { - if let cached = cache.object(forKey: imageId as NSString) { - return cached - } - if let running = inFlight[imageId] { - return await running.value - } - let api = api - let sessionId = sessionId - let task = Task { - guard let payload = try? await api.generatedImage(sessionId: sessionId, imageId: imageId) else { - return nil - } - let data = payload.data - return await Task.detached(priority: .utility) { - UIImage(data: data)?.preparingForDisplay() - }.value - } - inFlight[imageId] = task - let image = await task.value - inFlight[imageId] = nil - if let image { - cache.setObject(image, forKey: imageId as NSString) - } - return image - } - - func cachedImage(for imageId: String) -> UIImage? { - cache.object(forKey: imageId as NSString) - } -} - private struct ChatMediaKey: EnvironmentKey { - static let defaultValue: GeneratedImageLoader? = nil + static let defaultValue: GeneratedMediaLoader? = nil } extension EnvironmentValues { - /// The open chat's image loader; nil (previews/tests) degrades generated - /// images to a filename placeholder. - var chatMedia: GeneratedImageLoader? { + /// The open chat's authenticated image/file loader. + var chatMedia: GeneratedMediaLoader? { get { self[ChatMediaKey.self] } set { self[ChatMediaKey.self] = newValue } } diff --git a/ios/Hapi/Features/Chat/Blocks/GeneratedImageBlockView.swift b/ios/Hapi/Features/Chat/Blocks/GeneratedImageBlockView.swift index 11cc776a..a88c8d10 100644 --- a/ios/Hapi/Features/Chat/Blocks/GeneratedImageBlockView.swift +++ b/ios/Hapi/Features/Chat/Blocks/GeneratedImageBlockView.swift @@ -1,28 +1,49 @@ +import AVKit +import Combine import HapiProtocol +import QuickLook import SwiftUI import UIKit -/// Hub-generated image (`/api/sessions/:id/generated-images/:imageId`), -/// loaded as authed bytes through the chat's `GeneratedImageLoader` -/// (`\.chatMedia`; URLCache underneath honors the hub's immutable ETag). -/// Tap opens a simple full-screen viewer. Without a loader (previews/tests) -/// the card degrades to a filename placeholder. +/// The wire kind stays `generated-image` for all media. MIME, not the tool +/// name or filename, selects the renderer. Reset card state on row reuse. struct GeneratedImageBlockView: View { let block: GeneratedImageBlock - @Environment(\.chatMedia) private var media - @State private var image: UIImage? - @State private var failed = false - @State private var viewerOpen = false var body: some View { - Group { - if media == nil || failed { - placeholder - } else if let image = image ?? media?.cachedImage(for: block.imageId) { - Button { - viewerOpen = true - } label: { + GeneratedMediaCard(block: block, loader: media) + .id(block.imageId) + } +} + +private struct GeneratedMediaCard: View { + let loader: GeneratedMediaLoader? + @State private var content: GeneratedMediaPresentation + @State private var loadAttempt = 0 + @State private var imageOpen = false + @State private var previewOpen = false + @State private var shareOpen = false + @Environment(\.scenePhase) private var scenePhase + + init(block: GeneratedImageBlock, loader: GeneratedMediaLoader?) { + self.loader = loader + _content = State(initialValue: GeneratedMediaPresentation(block: block)) + } + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + Label { + Text(verbatim: content.block.fileName) + .lineLimit(2) + } icon: { + Image(systemName: content.kind.icon) + } + .font(.caption) + .foregroundStyle(.secondary) + + if let image = content.image { + Button { imageOpen = true } label: { Image(uiImage: image) .resizable() .scaledToFit() @@ -30,46 +51,138 @@ struct GeneratedImageBlockView: View { .clipShape(RoundedRectangle(cornerRadius: 12, style: .continuous)) } .buttonStyle(.plain) - } else { - loadingFrame - } - } - .task(id: block.imageId) { - guard let media, image == nil else { return } - if let loaded = await media.image(for: block.imageId) { - image = loaded - } else { - failed = true - } - } - .fullScreenCover(isPresented: $viewerOpen) { - ImageViewer(image: image, title: block.fileName) - } - } - - private var placeholder: some View { - Label(block.fileName, systemImage: "photo") - .font(.caption) - .foregroundStyle(.secondary) - .padding(.vertical, 4) - } - - private var loadingFrame: some View { - RoundedRectangle(cornerRadius: 12, style: .continuous) - .fill(Color.secondary.opacity(0.1)) - .frame(height: 160) - .overlay { + .accessibilityLabel(content.block.fileName) + } else if let file = content.file { + loadedFile(file) + } else if content.loadFailed { + Text("Couldn't load media.") + .font(.caption) + .foregroundStyle(.secondary) + Button("Retry", systemImage: "arrow.clockwise") { loadAttempt += 1 } + } else if loader != nil && (content.kind == .image || content.isLoading) { ProgressView() + .frame(maxWidth: .infinity, minHeight: 120) + } else if content.kind != .image { + loadButton + .disabled(loader == nil) } + } + .padding(12) + .background(Color(uiColor: .secondarySystemBackground), in: RoundedRectangle(cornerRadius: 12)) + .task(id: loadAttempt) { + guard let loader, content.kind == .image || loadAttempt > 0 else { return } + await content.load(using: loader) + } + .onDisappear { content.pause() } + .onChange(of: scenePhase) { _, phase in + if phase != .active { content.pause() } + } + .fullScreenCover(isPresented: $imageOpen) { + ImageViewer(image: content.image, title: content.block.fileName) + } + .sheet(isPresented: $previewOpen) { + if let file = content.file { + NavigationStack { + GeneratedMediaPreview(file: file) + .navigationTitle(content.block.fileName) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .confirmationAction) { + Button("Close") { previewOpen = false } + } + } + } + } + } + .sheet(isPresented: $shareOpen) { + if let file = content.file { GeneratedMediaShareSheet(file: file) } + } + } + + @ViewBuilder + private var loadButton: some View { + switch content.kind { + case .video: + Button("Load video", systemImage: "play.circle") { loadAttempt += 1 } + case .audio: + Button("Load audio", systemImage: "play.circle") { loadAttempt += 1 } + case .file: + Button("Prepare download", systemImage: "arrow.down.circle") { loadAttempt += 1 } + case .image: + EmptyView() + } + } + + @ViewBuilder + private func loadedFile(_ file: GeneratedMediaFile) -> some View { + if let player = content.player, let item = player.currentItem { + VideoPlayer(player: player) + .frame(height: content.kind == .audio ? 100 : 220) + .clipShape(RoundedRectangle(cornerRadius: 12)) + .onReceive(item.publisher(for: \.status)) { status in + if status == .failed { content.playbackFailed() } + } + } + if content.playbackUnavailable { + Text("This media cannot be played on this device.") + .font(.caption) + .foregroundStyle(.secondary) + } + ViewThatFits(in: .horizontal) { + HStack(spacing: 16) { fileActions(file) } + VStack(alignment: .leading, spacing: 12) { fileActions(file) } + } + } + + @ViewBuilder + private func fileActions(_ file: GeneratedMediaFile) -> some View { + if content.kind == .file, QLPreviewController.canPreview(file.url as NSURL) { + Button("Preview", systemImage: "doc.text.magnifyingglass") { previewOpen = true } + } + Button("Save or share", systemImage: "square.and.arrow.up") { shareOpen = true } } } -/// Minimal full-screen viewer: dark backdrop, fit-scaled image, tap or the -/// close button dismisses. (Pinch-zoom lands with the files feature.) +private struct GeneratedMediaShareSheet: UIViewControllerRepresentable { + let file: GeneratedMediaFile + + func makeUIViewController(context: Context) -> UIActivityViewController { + let controller = UIActivityViewController(activityItems: [file.url], applicationActivities: nil) + // Supply an anchor when UIKit presents the activity UI as an iPad popover. + controller.popoverPresentationController?.sourceView = controller.view + // The activity controller may outlive the transcript row/cache entry. + controller.completionWithItemsHandler = { [file] _, _, _, _ in _ = file } + return controller + } + + func updateUIViewController(_ controller: UIActivityViewController, context: Context) {} +} + +private struct GeneratedMediaPreview: UIViewControllerRepresentable { + let file: GeneratedMediaFile + + final class Coordinator: NSObject, QLPreviewControllerDataSource { + let file: GeneratedMediaFile + init(file: GeneratedMediaFile) { self.file = file } + func numberOfPreviewItems(in controller: QLPreviewController) -> Int { 1 } + func previewController(_ controller: QLPreviewController, previewItemAt index: Int) -> any QLPreviewItem { + file.url as NSURL + } + } + + func makeCoordinator() -> Coordinator { Coordinator(file: file) } + func makeUIViewController(context: Context) -> QLPreviewController { + let controller = QLPreviewController() + controller.dataSource = context.coordinator + return controller + } + func updateUIViewController(_ controller: QLPreviewController, context: Context) {} +} + +/// Minimal full-screen image viewer; keep the existing tap-to-dismiss behavior. private struct ImageViewer: View { let image: UIImage? let title: String - @Environment(\.dismiss) private var dismiss var body: some View { @@ -82,20 +195,16 @@ private struct ImageViewer: View { .frame(maxWidth: .infinity, maxHeight: .infinity) .padding(8) } else { - Text(title) - .foregroundStyle(.white) + Text(verbatim: title).foregroundStyle(.white) } - Button { - dismiss() - } label: { + Button { dismiss() } label: { Image(systemName: "xmark.circle.fill") .font(.title2) .foregroundStyle(.white.opacity(0.8)) .padding(16) } + .accessibilityLabel("Close") } - .onTapGesture { - dismiss() - } + .onTapGesture { dismiss() } } } diff --git a/ios/Hapi/Features/Chat/Blocks/GeneratedMediaLoader.swift b/ios/Hapi/Features/Chat/Blocks/GeneratedMediaLoader.swift new file mode 100644 index 00000000..9ab883da --- /dev/null +++ b/ios/Hapi/Features/Chat/Blocks/GeneratedMediaLoader.swift @@ -0,0 +1,110 @@ +import Foundation +import HapiClient +import UIKit +import UniformTypeIdentifiers + +/// Owns a private, non-user-selected path. References held by a player/share +/// sheet keep the file alive even after eviction from the per-chat cache. +final class GeneratedMediaFile: Sendable { + let url: URL + let byteCount: Int + private let directory: URL + + init(data: Data, fileName: String, mimeType: String?) throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("hapi-media-\(UUID().uuidString)", isDirectory: true) + // Neither the wire filename nor the media id may select a local path. + var name = String((fileName.replacingOccurrences(of: "\\", with: "/") + .components(separatedBy: "/").last ?? "").unicodeScalars + .filter { !CharacterSet.controlCharacters.contains($0) }) + if name.isEmpty || name == "." || name == ".." { name = "media" } + let suppliedExtension = (name as NSString).pathExtension + let mime = mimeType?.split(separator: ";", maxSplits: 1).first.map(String.init)?.lowercased() + let fallbackExtension = mime.flatMap { UTType(mimeType: $0)?.preferredFilenameExtension } ?? "" + let ext = suppliedExtension.isEmpty || suppliedExtension.utf8.count > 32 + ? fallbackExtension : suppliedExtension + var stem = (name as NSString).deletingPathExtension + let suffix = ext.isEmpty ? "" : ".\(ext)" + // APFS limits components by UTF-8 bytes, not Swift character count. + while stem.utf8.count + suffix.utf8.count > 220 { stem.removeLast() } + if stem.isEmpty { stem = "media" } + name = stem + suffix + url = directory.appendingPathComponent(name) + byteCount = data.count + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try data.write(to: url, options: .atomic) + } catch { + try? FileManager.default.removeItem(at: directory) + throw error + } + } + + deinit { + try? FileManager.default.removeItem(at: directory) + } +} + +/// Authenticated bytes (including JWT refresh) stay on APIClient, never on an +/// AVPlayer URL. Images decode off-main; other media is staged off-main to +/// private files for native playback/preview/sharing. Both paths single-flight. +@MainActor +final class GeneratedMediaLoader { + typealias Fetch = @Sendable (String) async throws -> (data: Data, mimeType: String?) + private let fetch: Fetch + private let images = NSCache() + private let files = NSCache() + private var imageTasks: [String: Task] = [:] + private var fileTasks: [String: Task] = [:] + + convenience init(api: APIClient, sessionId: String) { + self.init { imageId in + try await api.generatedImage(sessionId: sessionId, imageId: imageId) + } + } + + init(fetch: @escaping Fetch) { + self.fetch = fetch + images.countLimit = 40 + files.countLimit = 8 + files.totalCostLimit = 100 * 1024 * 1024 + } + + func image(for imageId: String) async throws -> UIImage { + if let cached = images.object(forKey: imageId as NSString) { return cached } + if let task = imageTasks[imageId] { return try await task.value } + let fetch = fetch + let task = Task { + let payload = try await fetch(imageId) + return try await Task.detached(priority: .utility) { + guard let image = UIImage(data: payload.data)?.preparingForDisplay() else { + throw CocoaError(.fileReadCorruptFile) + } + return image + }.value + } + imageTasks[imageId] = task + defer { imageTasks[imageId] = nil } + let image = try await task.value + images.setObject(image, forKey: imageId as NSString) + return image + } + + func file(for imageId: String, fileName: String, mimeType: String?) async throws -> GeneratedMediaFile { + if let cached = files.object(forKey: imageId as NSString) { return cached } + if let task = fileTasks[imageId] { return try await task.value } + let fetch = fetch + let task = Task { + let payload = try await fetch(imageId) + return try await Task.detached(priority: .utility) { + try GeneratedMediaFile(data: payload.data, fileName: fileName, + mimeType: mimeType ?? payload.mimeType) + }.value + } + fileTasks[imageId] = task + defer { fileTasks[imageId] = nil } + let file = try await task.value + files.setObject(file, forKey: imageId as NSString, cost: file.byteCount) + return file + } +} diff --git a/ios/Hapi/Features/Chat/Blocks/GeneratedMediaPresentation.swift b/ios/Hapi/Features/Chat/Blocks/GeneratedMediaPresentation.swift new file mode 100644 index 00000000..27d5e918 --- /dev/null +++ b/ios/Hapi/Features/Chat/Blocks/GeneratedMediaPresentation.swift @@ -0,0 +1,97 @@ +import AVFoundation +import HapiProtocol +import Observation +import UIKit + +enum GeneratedMediaKind { + case image, video, audio, file + + init(mimeType: String?) { + let mime = mimeType?.lowercased() ?? "image/" + if mime.hasPrefix("image/") { self = .image } + else if mime.hasPrefix("video/") { self = .video } + else if mime.hasPrefix("audio/") { self = .audio } + else { self = .file } + } + + var icon: String { + switch self { + case .image: "photo" + case .video: "video" + case .audio: "waveform" + case .file: "doc" + } + } +} + +/// One card's state, separate from the per-chat cache. Creating a model never +/// downloads anything: the view auto-loads only images, just like web. +@Observable @MainActor +final class GeneratedMediaPresentation { + let block: GeneratedImageBlock + let kind: GeneratedMediaKind + private(set) var image: UIImage? + private(set) var file: GeneratedMediaFile? + private(set) var player: AVPlayer? + private(set) var isLoading = false + private(set) var loadFailed = false + private(set) var playbackUnavailable = false + @ObservationIgnored private var loadGeneration = UUID() + + init(block: GeneratedImageBlock) { + self.block = block + kind = GeneratedMediaKind(mimeType: block.mimeType) + } + + func load(using loader: GeneratedMediaLoader) async { + guard image == nil, file == nil else { return } + // A row can reappear before its cancelled task finishes awaiting the + // shared download. Let the new task take over, not strand a spinner. + let generation = UUID() + loadGeneration = generation + isLoading = true + loadFailed = false + defer { + if loadGeneration == generation { isLoading = false } + } + do { + if kind == .image { + let image = try await loader.image(for: block.imageId) + try Task.checkCancellation() + guard loadGeneration == generation else { return } + self.image = image + } else { + let file = try await loader.file(for: block.imageId, fileName: block.fileName, mimeType: block.mimeType) + try Task.checkCancellation() + if kind == .video || kind == .audio { + let asset = AVURLAsset(url: file.url) + let playable = (try? await asset.load(.isPlayable)) ?? false + try Task.checkCancellation() + guard loadGeneration == generation else { return } + if playable { + // No autoplay. Native controls start playback explicitly. + player = AVPlayer(playerItem: AVPlayerItem(asset: asset)) + } else { + playbackUnavailable = true + } + } + guard loadGeneration == generation else { return } + self.file = file + } + } catch is CancellationError { + // A recycled/disappearing card must not publish a late result. + } catch { + if !Task.isCancelled, loadGeneration == generation { loadFailed = true } + } + } + + func playbackFailed() { + player?.pause() + player = nil + playbackUnavailable = true + } + + func pause() { + player?.pause() + } +} diff --git a/ios/Hapi/Features/Chat/ChatModel.swift b/ios/Hapi/Features/Chat/ChatModel.swift index 49606c2e..08b61a05 100644 --- a/ios/Hapi/Features/Chat/ChatModel.swift +++ b/ios/Hapi/Features/Chat/ChatModel.swift @@ -167,7 +167,7 @@ final class ChatModel { /// Transcripts append to the composer; failures surface as notices. let dictation: DictationController @ObservationIgnored private let pipeline = ChatPipeline() - @ObservationIgnored let imageLoader: GeneratedImageLoader + @ObservationIgnored let mediaLoader: GeneratedMediaLoader @ObservationIgnored let presentationState = ChatPresentationState() @ObservationIgnored let markdownCache = MarkdownRenderCache() @ObservationIgnored private var previousMarkdownSources: Set = [] @@ -205,7 +205,7 @@ final class ChatModel { api: session.api, recorder: AVAudioRecorderDictation() ) - self.imageLoader = GeneratedImageLoader(api: session.api, sessionId: sessionId) + self.mediaLoader = GeneratedMediaLoader(api: session.api, sessionId: sessionId) self.scratchlistAttachments = ScratchlistAttachmentLoader(api: session.api, sessionId: sessionId) self.header = ChatHeaderUI( title: String(sessionId.prefix(8)), diff --git a/ios/Hapi/Features/Chat/ChatView.swift b/ios/Hapi/Features/Chat/ChatView.swift index 80c444ad..0096a8e6 100644 --- a/ios/Hapi/Features/Chat/ChatView.swift +++ b/ios/Hapi/Features/Chat/ChatView.swift @@ -304,7 +304,7 @@ struct ChatView: View { .toolPresentations(model: model, session: session, owner: "chat") { path in viewerRoute = FileViewerRoute(sessionId: sessionId, path: path, mode: .file) } - .environment(\.chatMedia, model.imageLoader) + .environment(\.chatMedia, model.mediaLoader) .environment(\.chatPresentationState, model.presentationState) .environment(\.hapiMarkdownCache, model.markdownCache) .environment(\.chatInteractions, model.interactor) diff --git a/ios/Hapi/Features/Scratchlist/ScratchlistAttachmentLoader.swift b/ios/Hapi/Features/Scratchlist/ScratchlistAttachmentLoader.swift index 8064b3ff..07222b6d 100644 --- a/ios/Hapi/Features/Scratchlist/ScratchlistAttachmentLoader.swift +++ b/ios/Hapi/Features/Scratchlist/ScratchlistAttachmentLoader.swift @@ -3,7 +3,7 @@ import HapiClient import UIKit /// Per-chat scratchlist-attachment image loading (A-M4b), the -/// `GeneratedImageLoader` pattern: `GET /scratchlist/attachments/:id` bytes +/// `GeneratedMediaLoader` pattern: `GET /scratchlist/attachments/:id` bytes /// through the authed `APIClient` (URLCache underneath serves immutable /// repeats — attachment content never changes for an id), plus a small /// in-memory decode cache so list scrolling does not re-decode, with diff --git a/ios/Hapi/Resources/Localizable.xcstrings b/ios/Hapi/Resources/Localizable.xcstrings index 292e31ef..c18a68db 100644 --- a/ios/Hapi/Resources/Localizable.xcstrings +++ b/ios/Hapi/Resources/Localizable.xcstrings @@ -2950,6 +2950,23 @@ } } }, + "Couldn't load media." : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Не удалось загрузить медиа." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "无法加载媒体。" + } + } + } + }, "Couldn't load the scratchlist" : { "extractionState" : "manual", "localizations" : { @@ -5551,6 +5568,23 @@ } } }, + "Load audio" : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Загрузить аудио" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "加载音频" + } + } + } + }, "Load more content" : { "extractionState" : "manual", "localizations" : { @@ -5585,6 +5619,23 @@ } } }, + "Load video" : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Загрузить видео" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "加载视频" + } + } + } + }, "Loading machines…" : { "extractionState" : "manual", "localizations" : { @@ -7387,6 +7438,23 @@ } } }, + "Prepare download" : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Подготовить скачивание" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "准备下载" + } + } + } + }, "Preview" : { "extractionState" : "manual", "localizations" : { @@ -8560,6 +8628,23 @@ } } }, + "Save or share" : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Сохранить или поделиться" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "保存或分享" + } + } + } + }, "Scan Again" : { "extractionState" : "manual", "localizations" : { @@ -10175,6 +10260,23 @@ } } }, + "This media cannot be played on this device." : { + "extractionState" : "manual", + "localizations" : { + "ru" : { + "stringUnit" : { + "state" : "translated", + "value" : "Это медиа не воспроизводится на этом устройстве." + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "此设备无法播放该媒体。" + } + } + } + }, "This note's attachment budget is used up" : { "extractionState" : "manual", "localizations" : { diff --git a/ios/HapiTests/GeneratedMediaTests.swift b/ios/HapiTests/GeneratedMediaTests.swift new file mode 100644 index 00000000..cd70b9fc --- /dev/null +++ b/ios/HapiTests/GeneratedMediaTests.swift @@ -0,0 +1,277 @@ +import AVFoundation +import HapiClient +import SwiftUI +import UIKit +import XCTest +@testable import Hapi +@testable import HapiProtocol + +@MainActor +final class GeneratedMediaTests: XCTestCase { + private actor Source { + let data: Data + let mime: String + var calls = 0 + var failures: Int + + init(data: Data = Data("fixture".utf8), mime: String = "application/octet-stream", failures: Int = 0) { + self.data = data + self.mime = mime + self.failures = failures + } + + func fetch(_ id: String) async throws -> (data: Data, mimeType: String?) { + calls += 1 + try await Task.sleep(for: .milliseconds(30)) + if failures > 0 { + failures -= 1 + throw URLError(.timedOut) + } + return (data, mime) + } + } + + private func block(_ mime: String?, id: String = "media", name: String = "fixture") -> GeneratedImageBlock { + GeneratedImageBlock(id: id, localId: nil, createdAt: 0, invokedAt: nil, + imageId: id, fileName: name, mimeType: mime) + } + + func testMIMESelectsRendererNotFilenameOrToolName() { + XCTAssertEqual(GeneratedMediaKind(mimeType: nil), .image) + XCTAssertEqual(GeneratedMediaKind(mimeType: "image/png"), .image) + XCTAssertEqual(GeneratedMediaKind(mimeType: "video/mp4"), .video) + XCTAssertEqual(GeneratedMediaKind(mimeType: "video/webm"), .video) + XCTAssertEqual(GeneratedMediaKind(mimeType: "VIDEO/MP4"), .video) + XCTAssertEqual(GeneratedMediaKind(mimeType: "audio/wav"), .audio) + XCTAssertEqual(GeneratedMediaKind(mimeType: "application/pdf"), .file) + XCTAssertEqual(GeneratedMediaKind(mimeType: "application/octet-stream"), .file) + } + + func testUntouchedNonImageCardsDoNotFetchIncludingAfterReuse() async throws { + let source = Source() + let loader = GeneratedMediaLoader(fetch: source.fetch) + let scene = try XCTUnwrap(UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.first) + let window = UIWindow(windowScene: scene) + let host = UIHostingController(rootView: AnyView(EmptyView())) + window.rootViewController = host + window.makeKeyAndVisible() + defer { window.isHidden = true } + for (index, mime) in ["video/mp4", "audio/wav", "application/pdf"].enumerated() { + host.rootView = AnyView(GeneratedImageBlockView(block: block(mime, id: "\(index)")) + .environment(\.chatMedia, loader)) + host.view.layoutIfNeeded() + try await Task.sleep(for: .milliseconds(100)) + } + let calls = await source.calls + XCTAssertEqual(calls, 0) + } + + func testFilesSingleFlightCacheAndPreserveBytes() async throws { + let source = Source() + let loader = GeneratedMediaLoader(fetch: source.fetch) + async let first = loader.file(for: "id", fileName: "report.pdf", mimeType: "application/pdf") + async let second = loader.file(for: "id", fileName: "report.pdf", mimeType: "application/pdf") + let (a, b) = try await (first, second) + XCTAssertTrue(a === b) + let cached = try await loader.file(for: "id", fileName: "report.pdf", mimeType: "application/pdf") + XCTAssertTrue(a === cached) + XCTAssertEqual(try Data(contentsOf: a.url), Data("fixture".utf8)) + let calls = await source.calls + XCTAssertEqual(calls, 1) + } + + func testFailedDownloadCanBeRetried() async { + let source = Source(failures: 1) + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block("application/pdf")) + await content.load(using: loader) + XCTAssertTrue(content.loadFailed) + XCTAssertFalse(content.isLoading) + XCTAssertNil(content.file) + await content.load(using: loader) + XCTAssertFalse(content.loadFailed) + XCTAssertNotNil(content.file) + XCTAssertNil(content.player) + let calls = await source.calls + XCTAssertEqual(calls, 2) + } + + func testImageLoadingAndCachingStillWork() async throws { + let data = try XCTUnwrap(UIGraphicsImageRenderer(size: CGSize(width: 8, height: 8)).image { context in + UIColor.red.setFill() + context.fill(CGRect(x: 0, y: 0, width: 8, height: 8)) + }.pngData()) + let source = Source(data: data, mime: "image/png") + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block(nil)) + await content.load(using: loader) + XCTAssertNotNil(content.image) + XCTAssertFalse(content.loadFailed) + XCTAssertNil(content.file) + _ = try await loader.image(for: "media") + let calls = await source.calls + XCTAssertEqual(calls, 1) + } + + func testUnsupportedMediaKeepsFileForSharing() async { + // An unsupported/corrupt container must not fall back to an image placeholder. + let source = Source(data: Data("not a supported container".utf8), mime: "video/webm") + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block("video/webm", name: "clip.webm")) + await content.load(using: loader) + XCTAssertTrue(content.playbackUnavailable) + XCTAssertFalse(content.loadFailed) + XCTAssertNotNil(content.file) + XCTAssertNil(content.player) + } + + func testVideoAndAudioCreatePausedNativePlayersAndRetainShareFallback() async throws { + for (mime, name, data) in [("video/mp4", "clip.mp4", try await videoBytes()), + ("audio/wav", "speech.wav", audioBytes())] { + let source = Source(data: data, mime: mime) + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block(mime, name: name)) + await content.load(using: loader) + XCTAssertFalse(content.playbackUnavailable, mime) + let player = try XCTUnwrap(content.player, mime) + XCTAssertEqual(player.rate, 0, "No autoplay") + XCTAssertNotNil(content.file) + player.play() + content.pause() + XCTAssertEqual(player.rate, 0) + content.playbackFailed() + XCTAssertTrue(content.playbackUnavailable) + XCTAssertNil(content.player) + XCTAssertNotNil(content.file, "Late decoder errors must keep save/share available") + } + } + + func testCancelledLoadDoesNotPublishAndCanLoadAgainFromCache() async throws { + let source = Source() + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block("application/pdf")) + let loading = Task { await content.load(using: loader) } + for _ in 0..<100 { + if await source.calls > 0 { break } + try await Task.sleep(for: .milliseconds(5)) + } + loading.cancel() + await loading.value + XCTAssertNil(content.file) + XCTAssertFalse(content.loadFailed) + XCTAssertFalse(content.isLoading) + await content.load(using: loader) + XCTAssertNotNil(content.file) + let calls = await source.calls + XCTAssertEqual(calls, 1) + } + + func testReappearingCardTakesOverAnUnfinishedCancelledLoad() async throws { + let source = Source() + let loader = GeneratedMediaLoader(fetch: source.fetch) + let content = GeneratedMediaPresentation(block: block("application/pdf")) + let first = Task { await content.load(using: loader) } + for _ in 0..<100 { + if await source.calls > 0 { break } + try await Task.sleep(for: .milliseconds(1)) + } + first.cancel() + let reappeared = Task { await content.load(using: loader) } + await first.value + await reappeared.value + XCTAssertNotNil(content.file) + XCTAssertFalse(content.isLoading) + XCTAssertFalse(content.loadFailed) + let calls = await source.calls + XCTAssertEqual(calls, 1) + } + + func testTemporaryFilesUseSafeNamesAndAreRemovedOnRelease() throws { + var file: GeneratedMediaFile? = try GeneratedMediaFile( + data: Data("fixture".utf8), fileName: "../../outside.pdf", mimeType: "application/pdf") + let url = try XCTUnwrap(file?.url) + XCTAssertEqual(url.lastPathComponent, "outside.pdf") + XCTAssertTrue(url.deletingLastPathComponent().lastPathComponent.hasPrefix("hapi-media-")) + XCTAssertTrue(FileManager.default.fileExists(atPath: url.path)) + file = nil + XCTAssertFalse(FileManager.default.fileExists(atPath: url.deletingLastPathComponent().path)) + for name in ["..", "/", "..\\..\\", "\n"] { + let safe = try GeneratedMediaFile(data: Data(), fileName: name, mimeType: "video/mp4") + XCTAssertEqual(safe.url.lastPathComponent, "media.mp4") + } + let long = try GeneratedMediaFile(data: Data(), fileName: String(repeating: "视频", count: 100) + ".mp4", + mimeType: "video/mp4") + XCTAssertLessThanOrEqual(long.url.lastPathComponent.utf8.count, 220) + XCTAssertEqual(long.url.pathExtension, "mp4") + } + + func testCacheEvictionDoesNotDeleteAFileStillHeldByACard() async throws { + let source = Source() + var loader: GeneratedMediaLoader? = GeneratedMediaLoader(fetch: source.fetch) + var file: GeneratedMediaFile? = try await loader?.file(for: "id", fileName: "report", mimeType: "application/pdf") + let url = try XCTUnwrap(file?.url) + XCTAssertEqual(url.pathExtension, "pdf") + loader = nil + XCTAssertTrue(FileManager.default.fileExists(atPath: url.path)) + file = nil + XCTAssertFalse(FileManager.default.fileExists(atPath: url.path)) + } + + func testCardsHaveFiniteLayoutAtAccessibilitySizes() { + for mime in ["image/png", "video/mp4", "audio/wav", "application/pdf"] { + let host = UIHostingController(rootView: GeneratedImageBlockView(block: block(mime)) + .environment(\.dynamicTypeSize, .accessibility5)) + let size = host.sizeThatFits(in: CGSize(width: 280, height: CGFloat.greatestFiniteMagnitude)) + XCTAssertTrue(size.height.isFinite) + XCTAssertGreaterThan(size.height, 0) + XCTAssertLessThan(size.height, 600) + } + } + + private func audioBytes() -> Data { + let samples = 1_600 + var data = Data() + func text(_ string: String) { data.append(contentsOf: string.utf8) } + func number(_ value: T) { + var little = value.littleEndian + withUnsafeBytes(of: &little) { data.append(contentsOf: $0) } + } + text("RIFF"); number(UInt32(36 + samples * 2)); text("WAVEfmt ") + number(UInt32(16)); number(UInt16(1)); number(UInt16(1)) + number(UInt32(8_000)); number(UInt32(16_000)); number(UInt16(2)); number(UInt16(16)) + text("data"); number(UInt32(samples * 2)) + data.append(Data(count: samples * 2)) + return data + } + + private func videoBytes() async throws -> Data { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).mp4") + defer { try? FileManager.default.removeItem(at: url) } + let writer = try AVAssetWriter(outputURL: url, fileType: .mp4) + let input = AVAssetWriterInput(mediaType: .video, outputSettings: [ + AVVideoCodecKey: AVVideoCodecType.h264, AVVideoWidthKey: 32, AVVideoHeightKey: 32, + ]) + let adaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: input, sourcePixelBufferAttributes: nil) + writer.add(input) + XCTAssertTrue(writer.startWriting()) + writer.startSession(atSourceTime: .zero) + var buffer: CVPixelBuffer? + XCTAssertEqual(CVPixelBufferCreate(kCFAllocatorDefault, 32, 32, kCVPixelFormatType_32ARGB, + nil, &buffer), kCVReturnSuccess) + let pixel = try XCTUnwrap(buffer) + CVPixelBufferLockBaseAddress(pixel, []) + if let address = CVPixelBufferGetBaseAddress(pixel) { + memset(address, 0, CVPixelBufferGetDataSize(pixel)) + } + CVPixelBufferUnlockBaseAddress(pixel, []) + for _ in 0..<200 where !input.isReadyForMoreMediaData { + try await Task.sleep(for: .milliseconds(10)) + } + XCTAssertTrue(adaptor.append(pixel, withPresentationTime: .zero)) + writer.endSession(atSourceTime: CMTime(value: 1, timescale: 5)) + input.markAsFinished() + await writer.finishWriting() + XCTAssertEqual(writer.status, .completed, "\(String(describing: writer.error))") + return try Data(contentsOf: url) + } +} diff --git a/ios/README.md b/ios/README.md index 71464b1d..359d6369 100644 --- a/ios/README.md +++ b/ios/README.md @@ -20,6 +20,15 @@ archive (active) or reopen/delete (inactive), from the row's context menu or the chat's … menu; sending to an inactive session or reopening it can resume it and migrate the draft/navigation when the returned session ID changes. +Agent-displayed media (`display_image`, `display_video`, `display_media`) uses +MIME-based cards: images load automatically, while video, audio and other files +load only on request. Video/audio use native playback controls without autoplay; +leaving the row or backgrounding the app pauses playback. Loaded files can be +saved/shared, and supported documents open in Quick Look. Media that iOS cannot +decode (including unsupported WebM codecs) still offers save/share. Downloads use +the authenticated hub API, then private temporary files with a bounded per-chat +cache; the generated-media endpoint must still be available when first loading. + Model, permission and Codex collaboration controls follow the session's agent/capabilities; usage/storage require the owner namespace. See the [native app guide](../docs/guide/native-apps.md) for platform differences, web-only features and everyday use.