From: parabelboi Date: Wed, 12 Aug 2026 10:03:32 +0000 (+0200) Subject: ui: add read_media tool (#25877) X-Git-Tag: upstream/0.0.10438~51 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=4dd127584b87d1b12b6a33c2213197234503d8f7;p=pkg%2Fggml%2Fsources%2Fllama.cpp ui: add read_media tool (#25877) * server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the tag and removes the data URI before passing the tool result back to the model. * cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency * server: rename read_image tool to read_media for images and audio * Rename server_tool_read_image to server_tool_read_media in C++ * Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA * Rename UI constants, parser, and Svelte component files * Update display label from 'Read image' to 'Read media' * ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data * server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts * server: read_media: add to --tools help text and README tool list * ui: fix indentation in ChatMessageToolCallBlockDefault.svelte * server: read_media tool: fix a cast to use the correct type * server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts * server: make read_media inherit from read_file and add uses_cwd * ui: fix formating issues * rm from server * move it to frontend-only tool * correct partial commit * rm unused * ui: address review from allozaur Replace the magic strings, regexes and number in the read_media parser and service with named constants. Path splitting reuses FILE_PATH_SEPARATOR_REGEX, the size header regex moves to READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts. --------- Co-authored-by: ckrafft Co-authored-by: Xuan Son Nguyen Co-authored-by: Pascal --- diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index a4c1059ff..fd0ff8ddd 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1,6 +1,7 @@ #include "server-tools.h" #include "subproc.h" +#include "base64.hpp" #include #include @@ -864,6 +865,7 @@ static bool path_glob_match(const std::string & pattern, const std::string & rel // static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB +static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MB struct server_tool_read_file : server_tool { server_tool_read_file() { @@ -899,6 +901,8 @@ struct server_tool_read_file : server_tool { int start_line = json_value(params, "start_line", 1); int end_line = json_value(params, "end_line", -1); // -1 = no limit bool append_loc = json_value(params, "append_loc", false); + // comes from the x-resp-type header, the model cannot ask for it + bool as_base64 = json_value(params, "resp_type", std::string()) == "base64"; auto io = make_tools_io(params); @@ -906,6 +910,23 @@ struct server_tool_read_file : server_tool { if (!io->file_size(path, file_size)) { return {{"error", "cannot stat file: " + path}}; } + + if (as_base64) { + if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) { + return {{"error", string_format( + "file too large (%zu bytes, max %zu)", + (size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}}; + } + std::string content; + if (!io->read_file(path, content)) { + return {{"error", "failed to open file: " + path}}; + } + return { + {"base64", base64::encode(content.data(), content.size())}, + {"size_bytes", (size_t) content.size()}, + }; + } + if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) { return {{"error", string_format( "file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.", @@ -2135,6 +2156,15 @@ void server_tools::setup(const std::vector & enabled_tools, params["runtime"] = runtime->spec(); } + // x-resp-type header is only used by read_file for now + if (params.contains("resp_type")) { + params.erase("resp_type"); + } + auto resp_type = get_header(req.headers, "x-resp-type"); + if (!resp_type.empty()) { + params["resp_type"] = resp_type; + } + server_tool & tool = find_tool(tools, tool_name, stream); if (stream) { diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index 5bbaa4ea3..7a00f1a8e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -7,6 +7,7 @@ import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte'; import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte'; import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte'; + import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte'; import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte'; import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte'; import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte'; @@ -45,6 +46,8 @@ {:else if section.toolName === BuiltInTool.READ_FILE} +{:else if section.toolName === BuiltInTool.READ_MEDIA} + {:else if section.toolName === BuiltInTool.EDIT_FILE} {:else if section.toolName === BuiltInTool.WRITE_FILE} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index 34dfde78b..87208e2da 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -8,14 +8,16 @@ import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app'; import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants'; import { getBuiltinToolUi } from '$lib/constants/built-in-tools'; - import { FileTypeText, ToolResultKind } from '$lib/enums'; + import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums'; import type { DatabaseMessageExtra } from '$lib/types'; import { type AgenticSection, classifyToolResult, formatJsonPretty, - parseToolResultWithImages + parseToolResultWithMedia, + type ToolResultLine } from '$lib/utils'; + import { createBase64DataUrl } from '$lib/utils/data-url'; interface Props { section: AgenticSection; @@ -29,8 +31,8 @@ const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? ''); const outputKind = $derived(classifyToolResult(section.toolResult)); - const parsedLines = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + const parsedLines: ToolResultLine[] = $derived( + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); @@ -103,13 +105,26 @@
{line.text}
- {#if line.image} - {line.image.name} + {#if line.media} + {#if line.media.type === AttachmentType.AUDIO} + {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG} +
+ +
+ {:else} + {line.media.name} + {/if} {/if} {/each} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte index b2fa8b331..907a3cd12 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte @@ -23,7 +23,7 @@ isExitCodeSummaryLine, parseExecShellCommandError, parseExecShellCommandExitStatus, - parseToolResultWithImages, + parseToolResultWithMedia, type ToolResultLine } from '$lib/utils'; @@ -53,7 +53,7 @@ ); const parsedLines: ToolResultLine[] = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); // Drop the trailing "[exit code: N]" line - rendered as a colored @@ -223,10 +223,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.image} + {#if line.media} {line.image.name} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte new file mode 100644 index 000000000..424c794d0 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -0,0 +1,99 @@ + + + + {#snippet titleSnippet()} + Read media + {readMediaMeta?.fileName} + {/snippet} + + {#snippet children(_meta, _ctx)} + {#if section.toolResult} + {#if !mediaAttachment} +
+ Media attachment not found in message extras +
+ {:else if mediaAttachment.type === AttachmentType.AUDIO} +
+ +
+ {:else} +
+ {readMediaMeta?.fileName +
+ {/if} + + {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType} +
+ {#if readMediaMeta?.sizeBytes} + Size: {readMediaMeta.sizeBytes} bytes + {/if} + {#if readMediaMeta?.mimeType} + MIME: {readMediaMeta.mimeType} + {/if} +
+ {/if} + + {#if readMediaMeta?.path} +
{readMediaMeta.path}
+ {/if} + {:else} +
+ Waiting for media data... +
+ {/if} + {/snippet} + diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts new file mode 100644 index 000000000..ab1a4d6aa --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts @@ -0,0 +1,56 @@ +import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code'; +import { + PREFIX_FILE, + PREFIX_MIME, + PREFIX_SIZE, + READ_MEDIA_SIZE_REGEX +} from '$lib/constants/read-media'; +import type { AgenticSection } from '$lib/utils'; + +export interface ReadMediaMeta { + fileName: string; + path: string; + sizeBytes?: number; + mimeType?: string; +} + +/** + * Parse read_media tool result to extract metadata. + * Expected format (after extractBase64Attachments processing): + * File: /path/to/file.png + * Size: 12345 bytes + * MIME: image/png + * [Attachment saved: mcp-attachment-xxx.png] + * + * The data URI line is replaced by the attachment marker by + * agenticStore.extractBase64Attachments before storage. + */ +export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null { + if (!section.toolResult) return null; + + const lines = section.toolResult.split(NEWLINE); + + let fileName = ''; + let path = ''; + let sizeBytes: number | undefined; + let mimeType: string | undefined; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed.startsWith(PREFIX_FILE)) { + path = trimmed.slice(PREFIX_FILE.length).trim(); + fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path; + } else if (trimmed.startsWith(PREFIX_SIZE)) { + const match = trimmed.match(READ_MEDIA_SIZE_REGEX); + + if (match) sizeBytes = Number(match[1]); + } else if (trimmed.startsWith(PREFIX_MIME)) { + mimeType = trimmed.slice(PREFIX_MIME.length).trim(); + } + } + + if (!path) return null; + + return { fileName, mimeType, path, sizeBytes }; +} diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index 89bcbecb8..5bd24ffe8 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -10,6 +10,7 @@ import { Braces, Clock, + Eye, FilePen, FilePlus, FileSearch, @@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly> source: ToolSource.BUILTIN }, [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND }, [BuiltInTool.RUN_JAVASCRIPT]: { icon: Braces, label: 'Run JavaScript', diff --git a/tools/ui/src/lib/constants/code.ts b/tools/ui/src/lib/constants/code.ts index e57e1e6ec..4b4114200 100644 --- a/tools/ui/src/lib/constants/code.ts +++ b/tools/ui/src/lib/constants/code.ts @@ -18,6 +18,9 @@ export const TRIM_TRAILING_PADDING_REGEX = /(?:\n[ \t]*)+$/; // `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path. export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/; +// Separates a file name from its extension, e.g. the '.' in `cover.png`. +export const FILE_EXTENSION_SEPARATOR = '.'; + // Matches the `text:` prefix that file-type identifiers use to denote a // plain-text language (e.g. `text:typescript`). Used by tool-call renderers // to recover the underlying highlight.js language. diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 09006cf1e..357a33a62 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -49,6 +49,7 @@ export * from './sse'; export * from './precision'; export * from './processing-info'; export * from './pwa'; +export * from './read-media'; export * from './routes'; export * from './sandbox'; export * from './settings-keys'; diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index 0ef8d9ec1..c2639daa1 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -1,4 +1,4 @@ -import { MimeTypeImage } from '$lib/enums'; +import { MimeTypeAudio, MimeTypeImage } from '$lib/enums'; // File extension patterns for resource type detection export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i; @@ -27,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res'; // Default file extension for unknown image types export const DEFAULT_IMAGE_EXTENSION = 'img'; +// Default file extension for unknown audio types +export const DEFAULT_AUDIO_EXTENSION = 'mp3'; + // Default filename for resource content downloads export const DEFAULT_RESOURCE_FILENAME = 'resource.txt'; @@ -53,3 +56,18 @@ export const IMAGE_MIME_TO_EXTENSION: Record = { [MimeTypeImage.PNG]: 'png', [MimeTypeImage.WEBP]: 'webp' } as const; + +/** + * Mapping from audio MIME types to file extensions. + * Used for generating attachment filenames from MIME types. + */ +export const AUDIO_MIME_TO_EXTENSION: Record = { + [MimeTypeAudio.MP3]: 'mp3', + [MimeTypeAudio.MP3_MPEG]: 'mp3', + [MimeTypeAudio.VND_WAVE]: 'wav', + [MimeTypeAudio.WAV]: 'wav', + [MimeTypeAudio.WAVE]: 'wav', + [MimeTypeAudio.X_PN_WAV]: 'wav', + [MimeTypeAudio.X_WAV]: 'wav', + [MimeTypeAudio.X_WAVE]: 'wav' +} as const; diff --git a/tools/ui/src/lib/constants/read-media.ts b/tools/ui/src/lib/constants/read-media.ts new file mode 100644 index 000000000..525c5e902 --- /dev/null +++ b/tools/ui/src/lib/constants/read-media.ts @@ -0,0 +1,66 @@ +import { + BuiltInTool, + JsonSchemaType, + MimeTypeAudio, + MimeTypeImage, + ToolCallType +} from '$lib/enums'; +import type { OpenAIToolDefinition } from '$lib/types'; + +export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA; + +// header lines of the tool result, parsed back by the read_media renderer +export const PREFIX_FILE = 'File: '; +export const PREFIX_SIZE = 'Size: '; +export const PREFIX_MIME = 'MIME: '; + +/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */ +export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`); + +/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */ +export const READ_MEDIA_IMAGE_MIME: Record = { + gif: MimeTypeImage.GIF, + jpeg: MimeTypeImage.JPEG, + jpg: MimeTypeImage.JPEG, + png: MimeTypeImage.PNG +} as const; + +/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */ +export const READ_MEDIA_AUDIO_MIME: Record = { + mp3: MimeTypeAudio.MP3_MPEG, + wav: MimeTypeAudio.WAV +} as const; + +/** + * Build the read_media tool definition for the modalities the active model has. + * At least one of the two flags must be true, otherwise the tool is not offered + * at all - a model that cannot see or hear has nothing to do with the bytes. + */ +export function buildReadMediaToolDefinition( + supportsVision: boolean, + supportsAudio: boolean +): OpenAIToolDefinition { + const kinds: string[] = []; + + if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`); + + if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`); + + return { + function: { + description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`, + name: READ_MEDIA_TOOL_NAME, + parameters: { + properties: { + path: { + description: 'Path to the media file', + type: JsonSchemaType.STRING + } + }, + required: ['path'], + type: JsonSchemaType.OBJECT + } + }, + type: ToolCallType.FUNCTION + }; +} diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.ts index 65f4457c9..a467cda2b 100644 --- a/tools/ui/src/lib/constants/tools.ts +++ b/tools/ui/src/lib/constants/tools.ts @@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums'; /** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */ export const X_TOOL_CWD_HEADER = 'x-tool-cwd'; +/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */ +export const X_RESP_TYPE_HEADER = 'x-resp-type'; + +/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */ +export const RESP_TYPE_BASE64 = 'base64'; + export const TOOL_GROUP_LABELS = { [ToolSource.BUILTIN]: 'Built-in', [ToolSource.CUSTOM]: 'JSON Schema', diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts index eecb36c23..5785428cf 100644 --- a/tools/ui/src/lib/enums/files.enums.ts +++ b/tools/ui/src/lib/enums/files.enums.ts @@ -163,6 +163,7 @@ export enum FileExtensionText { // MIME type prefixes and includes for content detection export enum MimeTypePrefix { IMAGE = 'image/', + AUDIO = 'audio/', TEXT = 'text' } diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index 7a9751ee7..31c992fef 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -37,6 +37,7 @@ export enum GlobSearchType { */ export enum BuiltInTool { READ_FILE = 'read_file', + READ_MEDIA = 'read_media', EDIT_FILE = 'edit_file', WRITE_FILE = 'write_file', GET_DATETIME = 'get_datetime', diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 775a5d6e0..540d74239 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -1,4 +1,5 @@ import { settingsStore } from '../stores/settings.svelte'; +import { getAudioInputFormat } from '../utils/audio-format'; import { capImageDataURLSize } from '../utils/cap-img-size'; import { API_CHAT, @@ -20,18 +21,12 @@ import { import { AttachmentType, ContentPartType, - FileTypeAudio, MessageRole, - MimeTypeAudio, ReasoningFormat, StreamConnectionState } from '$lib/enums'; import { modelsStore } from '$lib/stores/models.svelte'; -import type { - AudioInputFormat, - DatabaseMessageExtraMcpPrompt, - DatabaseMessageExtraMcpResource -} from '$lib/types'; +import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types'; import type { ApiChatCompletionToolCall, ApiChatMessageContentPart, @@ -43,23 +38,6 @@ import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { streamIdentity } from '$lib/utils/stream-identity'; -function getAudioInputFormat(mimeType: string): AudioInputFormat { - const normalizedMimeType = mimeType.trim().toLowerCase(); - - if ( - normalizedMimeType === MimeTypeAudio.WAV || - normalizedMimeType === MimeTypeAudio.WAVE || - normalizedMimeType === MimeTypeAudio.X_WAV || - normalizedMimeType === MimeTypeAudio.X_WAVE || - normalizedMimeType === MimeTypeAudio.VND_WAVE || - normalizedMimeType === MimeTypeAudio.X_PN_WAV - ) { - return FileTypeAudio.WAV; - } - - return FileTypeAudio.MP3; -} - interface ResumableStreamState { bytesReceived: number; updatedAt: number; diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.ts new file mode 100644 index 000000000..fd66350d0 --- /dev/null +++ b/tools/ui/src/lib/services/read-media.service.ts @@ -0,0 +1,112 @@ +import { ToolsService } from './tools.service'; +import { + FILE_EXTENSION_SEPARATOR, + FILE_PATH_SEPARATOR_REGEX, + NEWLINE, + PREFIX_FILE, + PREFIX_MIME, + PREFIX_SIZE, + READ_MEDIA_AUDIO_MIME, + READ_MEDIA_IMAGE_MIME, + RESP_TYPE_BASE64 +} from '$lib/constants'; +import { BuiltInTool, ToolResponseField } from '$lib/enums'; +import type { ToolExecutionResult } from '$lib/types'; + +/** Modalities of the model the tool call runs for. */ +export interface ReadMediaCapabilities { + audio: boolean; + vision: boolean; +} + +/** Lowercase extension of a path, without the dot. Empty when the file name has none. */ +function fileExtension(path: string): string { + const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? ''; + const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR); + + return dot > 0 ? name.slice(dot + 1).toLowerCase() : ''; +} + +/** + * **ReadMediaService** - frontend executor for the `read_media` tool + * + * The tool is synthetic: no such tool exists on the server. It reads the file + * through the built-in `read_file` tool with the `base64` response type, then + * turns the bytes into a data URI line. The agentic store lifts that line into + * an image or audio attachment on the tool result message, which is what makes + * the model perceive the file instead of reading a wall of base64. + * + * Living in the frontend is what lets it exist only for models that can + * actually use the result - the server has no idea which model is selected. + * + * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM + * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction + */ +export class ReadMediaService { + static async executeTool( + params: Record, + capabilities: ReadMediaCapabilities, + signal?: AbortSignal, + cwd?: string + ): Promise { + const path = typeof params.path === 'string' ? params.path : ''; + + if (!path) { + return { content: 'Error: missing "path" argument.', isError: true }; + } + + const extension = fileExtension(path); + const imageMime = READ_MEDIA_IMAGE_MIME[extension]; + const audioMime = READ_MEDIA_AUDIO_MIME[extension]; + + let resolvedMime: string | undefined; + + if (imageMime && capabilities.vision) resolvedMime = imageMime; + else if (audioMime && capabilities.audio) resolvedMime = audioMime; + + if (!resolvedMime) { + const supported = [ + ...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []), + ...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : []) + ]; + // an unreadable-by-this-model file is a dead end, so say why instead of failing silently + const reason = + imageMime || audioMime + ? `the current model cannot perceive ".${extension}" files` + : `".${extension}" is not a supported media type`; + + return { + content: `Error: ${reason}. Supported: ${supported.join(', ')}.`, + isError: true + }; + } + + const raw = await ToolsService.executeToolRaw( + BuiltInTool.READ_FILE, + { path }, + signal, + cwd, + RESP_TYPE_BASE64 + ); + + if (ToolResponseField.ERROR in raw) { + return { content: String(raw[ToolResponseField.ERROR]), isError: true }; + } + + const base64 = typeof raw.base64 === 'string' ? raw.base64 : ''; + + if (!base64) { + return { content: `Error: no data returned for ${path}.`, isError: true }; + } + + const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0; + const content = [ + `${PREFIX_FILE}${path}`, + `${PREFIX_SIZE}${sizeBytes} bytes`, + `${PREFIX_MIME}${resolvedMime}`, + `data:${resolvedMime};base64,${base64}` + ].join(NEWLINE); + + return { content, isError: false }; + } +} diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.ts index cd6b12ceb..9cb5ecb46 100644 --- a/tools/ui/src/lib/services/tools.service.ts +++ b/tools/ui/src/lib/services/tools.service.ts @@ -1,5 +1,5 @@ import { base } from '$app/paths'; -import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants'; +import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants'; import { ToolResponseField } from '$lib/enums'; import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types'; import { apiFetch } from '$lib/utils'; @@ -51,16 +51,26 @@ export class ToolsService { * Execute a built-in tool and return the raw JSON response. Unlike * executeTool, this preserves structured fields (e.g. file_glob_search's * `entries` and `base`) that the flattened ToolExecutionResult drops. + * + * @param respType - sent as the x-resp-type request header. Only read_file + * honors it, with `base64` to get the raw bytes instead of decoded text. */ static async executeToolRaw( toolName: string, params: Record, signal?: AbortSignal, - cwd?: string + cwd?: string, + respType?: string ): Promise> { + const headers: Record = {}; + + if (cwd) headers[X_TOOL_CWD_HEADER] = cwd; + + if (respType) headers[X_RESP_TYPE_HEADER] = respType; + return apiFetch>(API_TOOLS.EXECUTE, { body: JSON.stringify({ params, tool: toolName }), - headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined, + headers: Object.keys(headers).length > 0 ? headers : undefined, method: 'POST', signal }); diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 080ed7c89..a84fa34dc 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -22,7 +22,9 @@ import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; import { + AUDIO_MIME_TO_EXTENSION, DATA_URI_BASE64_REGEX, + DEFAULT_AUDIO_EXTENSION, DEFAULT_IMAGE_EXTENSION, IMAGE_MIME_TO_EXTENSION, MCP_ATTACHMENT_NAME_PREFIX @@ -36,6 +38,7 @@ import { ToolCallType } from '$lib/enums'; import { ChatService } from '$lib/services'; +import { ReadMediaService } from '$lib/services/read-media.service'; import { SandboxService } from '$lib/services/sandbox.service'; import { ToolsService } from '$lib/services/tools.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; @@ -75,9 +78,10 @@ import type { import type { DatabaseMessage, DatabaseMessageExtra, + DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types/database'; -import { isAbortError } from '$lib/utils'; +import { getAudioInputFormat, isAbortError } from '$lib/utils'; import { SvelteMap } from 'svelte/reactivity'; function createDefaultSession(): AgenticSession { @@ -900,7 +904,18 @@ class AgenticStore { if (executionResult.isError) toolSuccess = false; } else if (toolSource === ToolSource.FRONTEND) { const args = this.parseToolArguments(toolCall.function.arguments); - const executionResult = await SandboxService.executeTool(toolName, args, signal); + const executionResult = + toolName === BuiltInTool.READ_MEDIA + ? await ReadMediaService.executeTool( + args, + { + audio: modelsStore.modelSupportsAudio(effectiveModel), + vision: modelsStore.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ) + : await SandboxService.executeTool(toolName, args, signal); result = executionResult.content; @@ -990,7 +1005,19 @@ class AgenticStore { ]; for (const attachment of attachments) { - if (attachment.type === AttachmentType.IMAGE) { + if (attachment.type === AttachmentType.AUDIO) { + if (modelsStore.modelSupportsAudio(effectiveModel)) { + contentParts.push({ + input_audio: { + data: (attachment as DatabaseMessageExtraAudioFile).base64Data, + format: getAudioInputFormat( + (attachment as DatabaseMessageExtraAudioFile).mimeType + ) + }, + type: ContentPartType.INPUT_AUDIO + }); + } + } else if (attachment.type === AttachmentType.IMAGE) { if (modelsStore.modelSupportsVision(effectiveModel)) { contentParts.push({ image_url: { @@ -1101,6 +1128,18 @@ class AgenticStore { return `[Attachment saved: ${name}]`; } + if (mimeType.startsWith(MimeTypePrefix.AUDIO)) { + // audio extras hold the bare base64, the input_audio part has no room for a data URI + attachments.push({ + base64Data, + mimeType, + name, + type: AttachmentType.AUDIO + }); + + return `[Attachment saved: ${name}]`; + } + return line; }); @@ -1108,7 +1147,9 @@ class AgenticStore { } private buildAttachmentName(mimeType: string, index: number): string { - const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION; + const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) + ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) + : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; } diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts index 145de119c..150fb3800 100644 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ b/tools/ui/src/lib/stores/models.svelte.ts @@ -18,7 +18,10 @@ import { ModelsService } from '$lib/services/models.service'; import { PropsService } from '$lib/services/props.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { isRouterMode, serverStore } from '$lib/stores/server.svelte'; -import { getAuthHeaders, TTLCache } from '$lib/utils'; +// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back +// into the stores, and going through it here would read a half-built module +import { getAuthHeaders } from '$lib/utils/api-headers'; +import { TTLCache } from '$lib/utils/cache-ttl'; import { detectThinkingSupport, detectThinkingSupportWithReason diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index b1a946391..3984dd2bc 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,4 +1,5 @@ import { + buildReadMediaToolDefinition, buildSandboxToolDefinition, DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, HOME_TILDE, @@ -15,6 +16,7 @@ import { } from '$lib/enums'; import { ToolsService } from '$lib/services/tools.service'; import { mcpStore } from '$lib/stores/mcp.svelte'; +import { modelsStore, selectedModelName } from '$lib/stores/models.svelte'; import { config } from '$lib/stores/settings.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -168,9 +170,42 @@ class ToolsStore { } get frontendTools(): OpenAIToolDefinition[] { - return config().jsSandboxEnabled - ? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)] - : []; + const tools: OpenAIToolDefinition[] = []; + + if (config().jsSandboxEnabled) { + tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled)); + } + + const readMedia = this.readMediaTool(); + + if (readMedia) tools.push(readMedia); + + return tools; + } + + /** + * `read_media` runs in the frontend on top of the server's `read_file`, so it + * exists only when that tool is served and the active model can perceive the + * bytes. The server cannot make this call - it does not know which model the + * conversation uses. + */ + private readMediaTool(): OpenAIToolDefinition | null { + const hasReadFile = this._builtinTools.some( + (def) => def.function.name === BuiltInTool.READ_FILE + ); + + if (!hasReadFile) return null; + + const model = selectedModelName() ?? modelsStore.models[0]?.model ?? ''; + + if (!model) return null; + + const vision = modelsStore.modelSupportsVision(model); + const audio = modelsStore.modelSupportsAudio(model); + + if (!vision && !audio) return null; + + return buildReadMediaToolDefinition(vision, audio); } get customTools(): OpenAIToolDefinition[] { diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index 9c92a42d4..b2dd2cd9e 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -50,11 +50,11 @@ export interface AgenticSection { } /** - * Represents a tool result line that may reference an image attachment + * Represents a tool result line that may reference a media attachment (image or audio) */ export type ToolResultLine = { text: string; - image?: DatabaseMessageExtraImageFile; + media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile; }; /** @@ -301,16 +301,16 @@ export function splitSearchSummaryList( return { lines }; } -/** Bounded cache for parseToolResultWithImages results. */ +/** Bounded cache for parseToolResultWithMedia results. */ const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32; const toolResultLinesCache = new Map(); /** - * Parse tool result text into lines, matching image attachments by name. + * Parse tool result text into lines, matching media attachments (images and audio) by name. * Memoized: called per render during streaming on unchanged tool result * strings with unchanged extras. */ -export function parseToolResultWithImages( +export function parseToolResultWithMedia( toolResult: string, extras?: DatabaseMessageExtra[] ): ToolResultLine[] { @@ -332,12 +332,13 @@ export function parseToolResultWithImages( if (!match || !extras) return { text: line }; const attachmentName = match[1]; - const image = extras.find( - (e): e is DatabaseMessageExtraImageFile => - e.type === AttachmentType.IMAGE && e.name === attachmentName + const media = extras.find( + (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile => + (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) && + e.name === attachmentName ); - return { image, text: line }; + return { media, text: line }; }); if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) { diff --git a/tools/ui/src/lib/utils/audio-format.ts b/tools/ui/src/lib/utils/audio-format.ts new file mode 100644 index 000000000..4f597aad3 --- /dev/null +++ b/tools/ui/src/lib/utils/audio-format.ts @@ -0,0 +1,22 @@ +import { FileTypeAudio, MimeTypeAudio } from '$lib/enums'; +import type { AudioInputFormat } from '$lib/types/api'; + +/** + * Map a MIME type to the AudioInputFormat expected by the API. + */ +export function getAudioInputFormat(mimeType: string): AudioInputFormat { + const normalizedMimeType = mimeType.trim().toLowerCase(); + + if ( + normalizedMimeType === MimeTypeAudio.WAV || + normalizedMimeType === MimeTypeAudio.WAVE || + normalizedMimeType === MimeTypeAudio.X_WAV || + normalizedMimeType === MimeTypeAudio.X_WAVE || + normalizedMimeType === MimeTypeAudio.VND_WAVE || + normalizedMimeType === MimeTypeAudio.X_PN_WAV + ) { + return FileTypeAudio.WAV; + } + + return FileTypeAudio.MP3; +} diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 5762e8e4b..c3585bd5f 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -248,7 +248,7 @@ export { export { deriveAgenticSections, buildAssistantRawOutput, - parseToolResultWithImages, + parseToolResultWithMedia, splitSearchSummaryList, hasAgenticContent, classifyToolResult, @@ -325,3 +325,6 @@ export { uuid } from './uuid'; // CSS utilities export { remToPx } from './css'; + +// Audio format helper (used by agentic store and chat service) +export { getAudioInputFormat } from './audio-format'; diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.md index a4e3e4c64..198b21956 100644 --- a/tools/ui/tests/client/README-perf.md +++ b/tools/ui/tests/client/README-perf.md @@ -33,12 +33,12 @@ npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.t The point of the harness is the _scaling curve_, not any single number. -| Knob | Reads on | -| --------------------------- | ---------------------------------------------------------------------------------------------------- | -| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | -| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). | -| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | -| `openCodeFence` | `hljs.highlightAuto` on partial code. | +| Knob | Reads on | +| --------------------------- | --------------------------------------------------------------------------------------------------- | +| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | +| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). | +| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | +| `openCodeFence` | `hljs.highlightAuto` on partial code. | Deliberately no hard assertions: CI timing is noisy and the value here is the before/after delta, not a gate. diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.ts index 8bdaa095a..24dcb4891 100644 --- a/tools/ui/tests/unit/agentic-hotpath.bench.ts +++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts @@ -6,7 +6,7 @@ // // Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts -import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic'; +import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic'; import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code'; import { computeLineDiff } from '$lib/utils/compute-line-diff'; import { preprocessLaTeX } from '$lib/utils/latex-protection'; @@ -200,17 +200,17 @@ describe('exit-code regex', () => { // --- per-line result parsers ---------------------------------------------- -describe('parseToolResultWithImages', () => { +describe('parseToolResultWithMedia', () => { bench('1KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_1KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_1KB, []); }); bench('200KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_200KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_200KB, []); }); bench('2MB', () => { - parseToolResultWithImages(SHELL_OUTPUT_2MB, []); + parseToolResultWithMedia(SHELL_OUTPUT_2MB, []); }); });