* server: add read_image tool (#25875)
Adds a server-tool that allows vision models to analyze server-side images.
This tool is reading a single file for now:
The image data is base64 encoded and passed to the UI, which
decodes it, fills the <img> tag and removes the data URI before
passing the tool result back to the model.
* cleanup read_image tool: move magic strings to constants
* Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts
with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants
* Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte
* Use NEWLINE constant from code.ts instead of hardcoded '\n'
* Use PREFIX_SIZE in regex pattern for size parsing
* Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp
to match the TypeScript PREFIX_* constants for consistency
* server: rename read_image tool to read_media for images and audio
* Rename server_tool_read_image to server_tool_read_media in C++
* Rename enum BuiltInTool.READ_IMAGE to READ_MEDIA
* Rename UI constants, parser, and Svelte component files
* Update display label from 'Read image' to 'Read media'
* ui: consolidate audio data URI handling into shared utility
* Extract getAudioInputFormat to a shared utility (was duplicated inline)
* Store raw base64 in base64Data on the message object
* Use base64Data to construct data URIs for audio rendering
* Update agentic store to build INPUT_AUDIO parts from base64Data
* server: read_media: restrict audio to wav/mp3 and minor fixes
* Server get_mime_from_extension now only advertises audio/wav and
audio/mpeg (the only formats the model's input_audio API accepts)
* Case-insensitive extension matching (fixes .MP3, .Wav, etc.)
* Unknown extensions return an error instead of a multi-MB data URI
that inflates model context with garbage
* Updated tool description to document supported formats
* Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server
* fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts
* server: read_media: add to --tools help text and README tool list
* ui: fix indentation in ChatMessageToolCallBlockDefault.svelte
* server: read_media tool: fix a cast to use the correct type
* server: read_media: multiple fixes
* server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file
* ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts
* server: make read_media inherit from read_file and add uses_cwd
* ui: fix formating issues
* rm from server
* move it to frontend-only tool
* correct partial commit
* rm unused
* ui: address review from allozaur
Replace the magic strings, regexes and number in the read_media parser
and service with named constants. Path splitting reuses
FILE_PATH_SEPARATOR_REGEX, the size header regex moves to
READ_MEDIA_SIZE_REGEX derived from PREFIX_SIZE, and
FILE_EXTENSION_SEPARATOR lands next to it in constants/code.ts.
---------
Co-authored-by: ckrafft <redacted>
Co-authored-by: Xuan Son Nguyen <redacted>
Co-authored-by: Pascal <redacted>
#include "server-tools.h"
#include "subproc.h"
+#include "base64.hpp"
#include <filesystem>
#include <fstream>
//
static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE = 16 * 1024; // 16 KB
+static constexpr size_t SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64 = 32 * 1024 * 1024; // 32 MB
struct server_tool_read_file : server_tool {
server_tool_read_file() {
int start_line = json_value(params, "start_line", 1);
int end_line = json_value(params, "end_line", -1); // -1 = no limit
bool append_loc = json_value(params, "append_loc", false);
+ // comes from the x-resp-type header, the model cannot ask for it
+ bool as_base64 = json_value(params, "resp_type", std::string()) == "base64";
auto io = make_tools_io(params);
if (!io->file_size(path, file_size)) {
return {{"error", "cannot stat file: " + path}};
}
+
+ if (as_base64) {
+ if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64) {
+ return {{"error", string_format(
+ "file too large (%zu bytes, max %zu)",
+ (size_t)file_size, SERVER_TOOL_READ_FILE_MAX_SIZE_BASE64)}};
+ }
+ std::string content;
+ if (!io->read_file(path, content)) {
+ return {{"error", "failed to open file: " + path}};
+ }
+ return {
+ {"base64", base64::encode(content.data(), content.size())},
+ {"size_bytes", (size_t) content.size()},
+ };
+ }
+
if (file_size > SERVER_TOOL_READ_FILE_MAX_SIZE && end_line == -1) {
return {{"error", string_format(
"file too large (%zu bytes, max %zu). Use start_line/end_line to read a portion.",
params["runtime"] = runtime->spec();
}
+ // x-resp-type header is only used by read_file for now
+ if (params.contains("resp_type")) {
+ params.erase("resp_type");
+ }
+ auto resp_type = get_header(req.headers, "x-resp-type");
+ if (!resp_type.empty()) {
+ params["resp_type"] = resp_type;
+ }
+
server_tool & tool = find_tool(tools, tool_name, stream);
if (stream) {
import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte';
import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte';
import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte';
+ import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte';
import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte';
import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';
import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';
<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />
{:else if section.toolName === BuiltInTool.READ_FILE}
<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />
+{:else if section.toolName === BuiltInTool.READ_MEDIA}
+ <ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />
{:else if section.toolName === BuiltInTool.EDIT_FILE}
<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />
{:else if section.toolName === BuiltInTool.WRITE_FILE}
import { MarkdownContent, SyntaxHighlightedCode } from '$lib/components/app';
import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
- import { FileTypeText, ToolResultKind } from '$lib/enums';
+ import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums';
import type { DatabaseMessageExtra } from '$lib/types';
import {
type AgenticSection,
classifyToolResult,
formatJsonPretty,
- parseToolResultWithImages
+ parseToolResultWithMedia,
+ type ToolResultLine
} from '$lib/utils';
+ import { createBase64DataUrl } from '$lib/utils/data-url';
interface Props {
section: AgenticSection;
const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
const outputKind = $derived(classifyToolResult(section.toolResult));
- const parsedLines = $derived(
- section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+ const parsedLines: ToolResultLine[] = $derived(
+ section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
);
</script>
<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">
{line.text}
</div>
- {#if line.image}
- <img
- src={line.image.base64Url}
- alt={line.image.name}
- class="mt-2 mb-2 h-auto max-w-full rounded-lg"
- loading="lazy"
- />
+ {#if line.media}
+ {#if line.media.type === AttachmentType.AUDIO}
+ {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG}
+ <div class="mt-2 mb-2">
+ <audio controls class="w-full rounded-lg">
+ <source
+ src={createBase64DataUrl(audioMimeType, line.media.base64Data)}
+ type={audioMimeType}
+ />
+ Your browser does not support the audio element.
+ </audio>
+ </div>
+ {:else}
+ <img
+ src={line.media.base64Url}
+ alt={line.media.name}
+ class="mt-2 mb-2 h-auto max-w-full rounded-lg"
+ loading="lazy"
+ />
+ {/if}
{/if}
{/each}
</div>
isExitCodeSummaryLine,
parseExecShellCommandError,
parseExecShellCommandExitStatus,
- parseToolResultWithImages,
+ parseToolResultWithMedia,
type ToolResultLine
} from '$lib/utils';
);
const parsedLines: ToolResultLine[] = $derived(
- section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+ section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
);
// Drop the trailing "[exit code: N]" line - rendered as a colored
>
{#each outputLines as line, i (i)}
<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>
- {#if line.image}
+ {#if line.media}
<img
- src={line.image.base64Url}
- alt={line.image.name}
+ src={line.media.base64Url}
+ alt={line.media.name}
class="mt-2 mb-2 h-auto max-w-full rounded-lg"
loading="lazy"
/>
--- /dev/null
+<script lang="ts">
+ import { parseReadMediaMeta } from './parsers/read-media';
+ import ToolCallBlock from './ToolCallBlock.svelte';
+ import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic';
+ import { AttachmentType, MimeTypeAudio } from '$lib/enums';
+ import type { DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types';
+ import { type AgenticSection } from '$lib/utils';
+ import { createBase64DataUrl } from '$lib/utils/data-url';
+
+ interface Props {
+ section: AgenticSection;
+ open: boolean;
+ isStreaming: boolean;
+ onToggle?: () => void;
+ }
+
+ let { isStreaming, onToggle, open, section }: Props = $props();
+
+ const readMediaMeta = $derived(parseReadMediaMeta(section));
+
+ // extractBase64Attachments swapped the data URI line for [Attachment saved: name]
+ // and moved the bytes to the message extras, so the name is the only link back
+ const mediaAttachment = $derived.by(() => {
+ const extras = section.toolResultExtras;
+
+ if (!extras || extras.length === 0) return null;
+
+ const match = section.toolResult?.match(ATTACHMENT_SAVED_REGEX);
+
+ if (!match) return null;
+
+ const attachmentName = match[1];
+
+ return (
+ extras.find(
+ (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+ (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+ e.name === attachmentName
+ ) ?? null
+ );
+ });
+
+ const audioMimeType = $derived(readMediaMeta?.mimeType ?? MimeTypeAudio.MP3_MPEG);
+</script>
+
+<ToolCallBlock {section} {open} {isStreaming} meta={readMediaMeta} {onToggle}>
+ {#snippet titleSnippet()}
+ <span class="text-muted-foreground">Read media </span>
+ <span class="font-mono">{readMediaMeta?.fileName}</span>
+ {/snippet}
+
+ {#snippet children(_meta, _ctx)}
+ {#if section.toolResult}
+ {#if !mediaAttachment}
+ <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+ Media attachment not found in message extras
+ </div>
+ {:else if mediaAttachment.type === AttachmentType.AUDIO}
+ <div class="mt-2">
+ <audio controls class="w-full rounded-lg">
+ <source
+ src={createBase64DataUrl(audioMimeType, mediaAttachment.base64Data)}
+ type={audioMimeType}
+ />
+ Your browser does not support the audio element.
+ </audio>
+ </div>
+ {:else}
+ <div class="mt-2">
+ <img
+ src={mediaAttachment.base64Url}
+ alt={readMediaMeta?.fileName ?? 'media'}
+ class="max-h-[60vh] max-w-full rounded-lg object-contain shadow-lg"
+ loading="lazy"
+ />
+ </div>
+ {/if}
+
+ {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}
+ <div class="mt-2 flex gap-4 text-xs text-muted-foreground">
+ {#if readMediaMeta?.sizeBytes}
+ <span>Size: {readMediaMeta.sizeBytes} bytes</span>
+ {/if}
+ {#if readMediaMeta?.mimeType}
+ <span>MIME: {readMediaMeta.mimeType}</span>
+ {/if}
+ </div>
+ {/if}
+
+ {#if readMediaMeta?.path}
+ <div class="mt-1 font-mono text-xs text-muted-foreground/60">{readMediaMeta.path}</div>
+ {/if}
+ {:else}
+ <div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+ Waiting for media data...
+ </div>
+ {/if}
+ {/snippet}
+</ToolCallBlock>
--- /dev/null
+import { FILE_PATH_SEPARATOR_REGEX, NEWLINE } from '$lib/constants/code';
+import {
+ PREFIX_FILE,
+ PREFIX_MIME,
+ PREFIX_SIZE,
+ READ_MEDIA_SIZE_REGEX
+} from '$lib/constants/read-media';
+import type { AgenticSection } from '$lib/utils';
+
+export interface ReadMediaMeta {
+ fileName: string;
+ path: string;
+ sizeBytes?: number;
+ mimeType?: string;
+}
+
+/**
+ * Parse read_media tool result to extract metadata.
+ * Expected format (after extractBase64Attachments processing):
+ * File: /path/to/file.png
+ * Size: 12345 bytes
+ * MIME: image/png
+ * [Attachment saved: mcp-attachment-xxx.png]
+ *
+ * The data URI line is replaced by the attachment marker by
+ * agenticStore.extractBase64Attachments before storage.
+ */
+export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null {
+ if (!section.toolResult) return null;
+
+ const lines = section.toolResult.split(NEWLINE);
+
+ let fileName = '';
+ let path = '';
+ let sizeBytes: number | undefined;
+ let mimeType: string | undefined;
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+
+ if (trimmed.startsWith(PREFIX_FILE)) {
+ path = trimmed.slice(PREFIX_FILE.length).trim();
+ fileName = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? path;
+ } else if (trimmed.startsWith(PREFIX_SIZE)) {
+ const match = trimmed.match(READ_MEDIA_SIZE_REGEX);
+
+ if (match) sizeBytes = Number(match[1]);
+ } else if (trimmed.startsWith(PREFIX_MIME)) {
+ mimeType = trimmed.slice(PREFIX_MIME.length).trim();
+ }
+ }
+
+ if (!path) return null;
+
+ return { fileName, mimeType, path, sizeBytes };
+}
import {
Braces,
Clock,
+ Eye,
FilePen,
FilePlus,
FileSearch,
source: ToolSource.BUILTIN
},
[BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN },
+ [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND },
[BuiltInTool.RUN_JAVASCRIPT]: {
icon: Braces,
label: 'Run JavaScript',
// `C:\foo\bar.txt`. Used wherever a parameter accepts a user-supplied path.
export const FILE_PATH_SEPARATOR_REGEX = /[\\/]/;
+// Separates a file name from its extension, e.g. the '.' in `cover.png`.
+export const FILE_EXTENSION_SEPARATOR = '.';
+
// Matches the `text:` prefix that file-type identifiers use to denote a
// plain-text language (e.g. `text:typescript`). Used by tool-call renderers
// to recover the underlying highlight.js language.
export * from './precision';
export * from './processing-info';
export * from './pwa';
+export * from './read-media';
export * from './routes';
export * from './sandbox';
export * from './settings-keys';
-import { MimeTypeImage } from '$lib/enums';
+import { MimeTypeAudio, MimeTypeImage } from '$lib/enums';
// File extension patterns for resource type detection
export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i;
// Default file extension for unknown image types
export const DEFAULT_IMAGE_EXTENSION = 'img';
+// Default file extension for unknown audio types
+export const DEFAULT_AUDIO_EXTENSION = 'mp3';
+
// Default filename for resource content downloads
export const DEFAULT_RESOURCE_FILENAME = 'resource.txt';
[MimeTypeImage.PNG]: 'png',
[MimeTypeImage.WEBP]: 'webp'
} as const;
+
+/**
+ * Mapping from audio MIME types to file extensions.
+ * Used for generating attachment filenames from MIME types.
+ */
+export const AUDIO_MIME_TO_EXTENSION: Record<string, string> = {
+ [MimeTypeAudio.MP3]: 'mp3',
+ [MimeTypeAudio.MP3_MPEG]: 'mp3',
+ [MimeTypeAudio.VND_WAVE]: 'wav',
+ [MimeTypeAudio.WAV]: 'wav',
+ [MimeTypeAudio.WAVE]: 'wav',
+ [MimeTypeAudio.X_PN_WAV]: 'wav',
+ [MimeTypeAudio.X_WAV]: 'wav',
+ [MimeTypeAudio.X_WAVE]: 'wav'
+} as const;
--- /dev/null
+import {
+ BuiltInTool,
+ JsonSchemaType,
+ MimeTypeAudio,
+ MimeTypeImage,
+ ToolCallType
+} from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA;
+
+// header lines of the tool result, parsed back by the read_media renderer
+export const PREFIX_FILE = 'File: ';
+export const PREFIX_SIZE = 'Size: ';
+export const PREFIX_MIME = 'MIME: ';
+
+/** Byte count of the `Size: ` header line, e.g. `Size: 12345 bytes` -> capture group 1 is `12345`. */
+export const READ_MEDIA_SIZE_REGEX = new RegExp(`^${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`);
+
+/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */
+export const READ_MEDIA_IMAGE_MIME: Record<string, string> = {
+ gif: MimeTypeImage.GIF,
+ jpeg: MimeTypeImage.JPEG,
+ jpg: MimeTypeImage.JPEG,
+ png: MimeTypeImage.PNG
+} as const;
+
+/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */
+export const READ_MEDIA_AUDIO_MIME: Record<string, string> = {
+ mp3: MimeTypeAudio.MP3_MPEG,
+ wav: MimeTypeAudio.WAV
+} as const;
+
+/**
+ * Build the read_media tool definition for the modalities the active model has.
+ * At least one of the two flags must be true, otherwise the tool is not offered
+ * at all - a model that cannot see or hear has nothing to do with the bytes.
+ */
+export function buildReadMediaToolDefinition(
+ supportsVision: boolean,
+ supportsAudio: boolean
+): OpenAIToolDefinition {
+ const kinds: string[] = [];
+
+ if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`);
+
+ if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`);
+
+ return {
+ function: {
+ description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`,
+ name: READ_MEDIA_TOOL_NAME,
+ parameters: {
+ properties: {
+ path: {
+ description: 'Path to the media file',
+ type: JsonSchemaType.STRING
+ }
+ },
+ required: ['path'],
+ type: JsonSchemaType.OBJECT
+ }
+ },
+ type: ToolCallType.FUNCTION
+ };
+}
/** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */
export const X_TOOL_CWD_HEADER = 'x-tool-cwd';
+/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */
+export const X_RESP_TYPE_HEADER = 'x-resp-type';
+
+/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */
+export const RESP_TYPE_BASE64 = 'base64';
+
export const TOOL_GROUP_LABELS = {
[ToolSource.BUILTIN]: 'Built-in',
[ToolSource.CUSTOM]: 'JSON Schema',
// MIME type prefixes and includes for content detection
export enum MimeTypePrefix {
IMAGE = 'image/',
+ AUDIO = 'audio/',
TEXT = 'text'
}
*/
export enum BuiltInTool {
READ_FILE = 'read_file',
+ READ_MEDIA = 'read_media',
EDIT_FILE = 'edit_file',
WRITE_FILE = 'write_file',
GET_DATETIME = 'get_datetime',
import { settingsStore } from '../stores/settings.svelte';
+import { getAudioInputFormat } from '../utils/audio-format';
import { capImageDataURLSize } from '../utils/cap-img-size';
import {
API_CHAT,
import {
AttachmentType,
ContentPartType,
- FileTypeAudio,
MessageRole,
- MimeTypeAudio,
ReasoningFormat,
StreamConnectionState
} from '$lib/enums';
import { modelsStore } from '$lib/stores/models.svelte';
-import type {
- AudioInputFormat,
- DatabaseMessageExtraMcpPrompt,
- DatabaseMessageExtraMcpResource
-} from '$lib/types';
+import type { DatabaseMessageExtraMcpPrompt, DatabaseMessageExtraMcpResource } from '$lib/types';
import type {
ApiChatCompletionToolCall,
ApiChatMessageContentPart,
import { formatAttachmentText } from '$lib/utils/formatters';
import { streamIdentity } from '$lib/utils/stream-identity';
-function getAudioInputFormat(mimeType: string): AudioInputFormat {
- const normalizedMimeType = mimeType.trim().toLowerCase();
-
- if (
- normalizedMimeType === MimeTypeAudio.WAV ||
- normalizedMimeType === MimeTypeAudio.WAVE ||
- normalizedMimeType === MimeTypeAudio.X_WAV ||
- normalizedMimeType === MimeTypeAudio.X_WAVE ||
- normalizedMimeType === MimeTypeAudio.VND_WAVE ||
- normalizedMimeType === MimeTypeAudio.X_PN_WAV
- ) {
- return FileTypeAudio.WAV;
- }
-
- return FileTypeAudio.MP3;
-}
-
interface ResumableStreamState {
bytesReceived: number;
updatedAt: number;
--- /dev/null
+import { ToolsService } from './tools.service';
+import {
+ FILE_EXTENSION_SEPARATOR,
+ FILE_PATH_SEPARATOR_REGEX,
+ NEWLINE,
+ PREFIX_FILE,
+ PREFIX_MIME,
+ PREFIX_SIZE,
+ READ_MEDIA_AUDIO_MIME,
+ READ_MEDIA_IMAGE_MIME,
+ RESP_TYPE_BASE64
+} from '$lib/constants';
+import { BuiltInTool, ToolResponseField } from '$lib/enums';
+import type { ToolExecutionResult } from '$lib/types';
+
+/** Modalities of the model the tool call runs for. */
+export interface ReadMediaCapabilities {
+ audio: boolean;
+ vision: boolean;
+}
+
+/** Lowercase extension of a path, without the dot. Empty when the file name has none. */
+function fileExtension(path: string): string {
+ const name = path.split(FILE_PATH_SEPARATOR_REGEX).pop() ?? '';
+ const dot = name.lastIndexOf(FILE_EXTENSION_SEPARATOR);
+
+ return dot > 0 ? name.slice(dot + 1).toLowerCase() : '';
+}
+
+/**
+ * **ReadMediaService** - frontend executor for the `read_media` tool
+ *
+ * The tool is synthetic: no such tool exists on the server. It reads the file
+ * through the built-in `read_file` tool with the `base64` response type, then
+ * turns the bytes into a data URI line. The agentic store lifts that line into
+ * an image or audio attachment on the tool result message, which is what makes
+ * the model perceive the file instead of reading a wall of base64.
+ *
+ * Living in the frontend is what lets it exist only for models that can
+ * actually use the result - the server has no idea which model is selected.
+ *
+ * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM
+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction
+ */
+export class ReadMediaService {
+ static async executeTool(
+ params: Record<string, unknown>,
+ capabilities: ReadMediaCapabilities,
+ signal?: AbortSignal,
+ cwd?: string
+ ): Promise<ToolExecutionResult> {
+ const path = typeof params.path === 'string' ? params.path : '';
+
+ if (!path) {
+ return { content: 'Error: missing "path" argument.', isError: true };
+ }
+
+ const extension = fileExtension(path);
+ const imageMime = READ_MEDIA_IMAGE_MIME[extension];
+ const audioMime = READ_MEDIA_AUDIO_MIME[extension];
+
+ let resolvedMime: string | undefined;
+
+ if (imageMime && capabilities.vision) resolvedMime = imageMime;
+ else if (audioMime && capabilities.audio) resolvedMime = audioMime;
+
+ if (!resolvedMime) {
+ const supported = [
+ ...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []),
+ ...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : [])
+ ];
+ // an unreadable-by-this-model file is a dead end, so say why instead of failing silently
+ const reason =
+ imageMime || audioMime
+ ? `the current model cannot perceive ".${extension}" files`
+ : `".${extension}" is not a supported media type`;
+
+ return {
+ content: `Error: ${reason}. Supported: ${supported.join(', ')}.`,
+ isError: true
+ };
+ }
+
+ const raw = await ToolsService.executeToolRaw(
+ BuiltInTool.READ_FILE,
+ { path },
+ signal,
+ cwd,
+ RESP_TYPE_BASE64
+ );
+
+ if (ToolResponseField.ERROR in raw) {
+ return { content: String(raw[ToolResponseField.ERROR]), isError: true };
+ }
+
+ const base64 = typeof raw.base64 === 'string' ? raw.base64 : '';
+
+ if (!base64) {
+ return { content: `Error: no data returned for ${path}.`, isError: true };
+ }
+
+ const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0;
+ const content = [
+ `${PREFIX_FILE}${path}`,
+ `${PREFIX_SIZE}${sizeBytes} bytes`,
+ `${PREFIX_MIME}${resolvedMime}`,
+ `data:${resolvedMime};base64,${base64}`
+ ].join(NEWLINE);
+
+ return { content, isError: false };
+ }
+}
import { base } from '$app/paths';
-import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants';
+import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants';
import { ToolResponseField } from '$lib/enums';
import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types';
import { apiFetch } from '$lib/utils';
* Execute a built-in tool and return the raw JSON response. Unlike
* executeTool, this preserves structured fields (e.g. file_glob_search's
* `entries` and `base`) that the flattened ToolExecutionResult drops.
+ *
+ * @param respType - sent as the x-resp-type request header. Only read_file
+ * honors it, with `base64` to get the raw bytes instead of decoded text.
*/
static async executeToolRaw(
toolName: string,
params: Record<string, unknown>,
signal?: AbortSignal,
- cwd?: string
+ cwd?: string,
+ respType?: string
): Promise<Record<string, unknown>> {
+ const headers: Record<string, string> = {};
+
+ if (cwd) headers[X_TOOL_CWD_HEADER] = cwd;
+
+ if (respType) headers[X_RESP_TYPE_HEADER] = respType;
+
return apiFetch<Record<string, unknown>>(API_TOOLS.EXECUTE, {
body: JSON.stringify({ params, tool: toolName }),
- headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined,
+ headers: Object.keys(headers).length > 0 ? headers : undefined,
method: 'POST',
signal
});
import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants';
import {
+ AUDIO_MIME_TO_EXTENSION,
DATA_URI_BASE64_REGEX,
+ DEFAULT_AUDIO_EXTENSION,
DEFAULT_IMAGE_EXTENSION,
IMAGE_MIME_TO_EXTENSION,
MCP_ATTACHMENT_NAME_PREFIX
ToolCallType
} from '$lib/enums';
import { ChatService } from '$lib/services';
+import { ReadMediaService } from '$lib/services/read-media.service';
import { SandboxService } from '$lib/services/sandbox.service';
import { ToolsService } from '$lib/services/tools.service';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import type {
DatabaseMessage,
DatabaseMessageExtra,
+ DatabaseMessageExtraAudioFile,
DatabaseMessageExtraImageFile
} from '$lib/types/database';
-import { isAbortError } from '$lib/utils';
+import { getAudioInputFormat, isAbortError } from '$lib/utils';
import { SvelteMap } from 'svelte/reactivity';
function createDefaultSession(): AgenticSession {
if (executionResult.isError) toolSuccess = false;
} else if (toolSource === ToolSource.FRONTEND) {
const args = this.parseToolArguments(toolCall.function.arguments);
- const executionResult = await SandboxService.executeTool(toolName, args, signal);
+ const executionResult =
+ toolName === BuiltInTool.READ_MEDIA
+ ? await ReadMediaService.executeTool(
+ args,
+ {
+ audio: modelsStore.modelSupportsAudio(effectiveModel),
+ vision: modelsStore.modelSupportsVision(effectiveModel)
+ },
+ signal,
+ conversationsStore.activeConversation?.cwd
+ )
+ : await SandboxService.executeTool(toolName, args, signal);
result = executionResult.content;
];
for (const attachment of attachments) {
- if (attachment.type === AttachmentType.IMAGE) {
+ if (attachment.type === AttachmentType.AUDIO) {
+ if (modelsStore.modelSupportsAudio(effectiveModel)) {
+ contentParts.push({
+ input_audio: {
+ data: (attachment as DatabaseMessageExtraAudioFile).base64Data,
+ format: getAudioInputFormat(
+ (attachment as DatabaseMessageExtraAudioFile).mimeType
+ )
+ },
+ type: ContentPartType.INPUT_AUDIO
+ });
+ }
+ } else if (attachment.type === AttachmentType.IMAGE) {
if (modelsStore.modelSupportsVision(effectiveModel)) {
contentParts.push({
image_url: {
return `[Attachment saved: ${name}]`;
}
+ if (mimeType.startsWith(MimeTypePrefix.AUDIO)) {
+ // audio extras hold the bare base64, the input_audio part has no room for a data URI
+ attachments.push({
+ base64Data,
+ mimeType,
+ name,
+ type: AttachmentType.AUDIO
+ });
+
+ return `[Attachment saved: ${name}]`;
+ }
+
return line;
});
}
private buildAttachmentName(mimeType: string, index: number): string {
- const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION;
+ const extension = mimeType.startsWith(MimeTypePrefix.AUDIO)
+ ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION)
+ : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION);
return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`;
}
import { PropsService } from '$lib/services/props.service';
import { conversationsStore } from '$lib/stores/conversations.svelte';
import { isRouterMode, serverStore } from '$lib/stores/server.svelte';
-import { getAuthHeaders, TTLCache } from '$lib/utils';
+// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back
+// into the stores, and going through it here would read a half-built module
+import { getAuthHeaders } from '$lib/utils/api-headers';
+import { TTLCache } from '$lib/utils/cache-ttl';
import {
detectThinkingSupport,
detectThinkingSupportWithReason
import {
+ buildReadMediaToolDefinition,
buildSandboxToolDefinition,
DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
HOME_TILDE,
} from '$lib/enums';
import { ToolsService } from '$lib/services/tools.service';
import { mcpStore } from '$lib/stores/mcp.svelte';
+import { modelsStore, selectedModelName } from '$lib/stores/models.svelte';
import { config } from '$lib/stores/settings.svelte';
import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types';
import { SvelteMap, SvelteSet } from 'svelte/reactivity';
}
get frontendTools(): OpenAIToolDefinition[] {
- return config().jsSandboxEnabled
- ? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)]
- : [];
+ const tools: OpenAIToolDefinition[] = [];
+
+ if (config().jsSandboxEnabled) {
+ tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled));
+ }
+
+ const readMedia = this.readMediaTool();
+
+ if (readMedia) tools.push(readMedia);
+
+ return tools;
+ }
+
+ /**
+ * `read_media` runs in the frontend on top of the server's `read_file`, so it
+ * exists only when that tool is served and the active model can perceive the
+ * bytes. The server cannot make this call - it does not know which model the
+ * conversation uses.
+ */
+ private readMediaTool(): OpenAIToolDefinition | null {
+ const hasReadFile = this._builtinTools.some(
+ (def) => def.function.name === BuiltInTool.READ_FILE
+ );
+
+ if (!hasReadFile) return null;
+
+ const model = selectedModelName() ?? modelsStore.models[0]?.model ?? '';
+
+ if (!model) return null;
+
+ const vision = modelsStore.modelSupportsVision(model);
+ const audio = modelsStore.modelSupportsAudio(model);
+
+ if (!vision && !audio) return null;
+
+ return buildReadMediaToolDefinition(vision, audio);
}
get customTools(): OpenAIToolDefinition[] {
}
/**
- * Represents a tool result line that may reference an image attachment
+ * Represents a tool result line that may reference a media attachment (image or audio)
*/
export type ToolResultLine = {
text: string;
- image?: DatabaseMessageExtraImageFile;
+ media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile;
};
/**
return { lines };
}
-/** Bounded cache for parseToolResultWithImages results. */
+/** Bounded cache for parseToolResultWithMedia results. */
const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32;
const toolResultLinesCache = new Map<string, ToolResultLine[]>();
/**
- * Parse tool result text into lines, matching image attachments by name.
+ * Parse tool result text into lines, matching media attachments (images and audio) by name.
* Memoized: called per render during streaming on unchanged tool result
* strings with unchanged extras.
*/
-export function parseToolResultWithImages(
+export function parseToolResultWithMedia(
toolResult: string,
extras?: DatabaseMessageExtra[]
): ToolResultLine[] {
if (!match || !extras) return { text: line };
const attachmentName = match[1];
- const image = extras.find(
- (e): e is DatabaseMessageExtraImageFile =>
- e.type === AttachmentType.IMAGE && e.name === attachmentName
+ const media = extras.find(
+ (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+ (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+ e.name === attachmentName
);
- return { image, text: line };
+ return { media, text: line };
});
if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) {
--- /dev/null
+import { FileTypeAudio, MimeTypeAudio } from '$lib/enums';
+import type { AudioInputFormat } from '$lib/types/api';
+
+/**
+ * Map a MIME type to the AudioInputFormat expected by the API.
+ */
+export function getAudioInputFormat(mimeType: string): AudioInputFormat {
+ const normalizedMimeType = mimeType.trim().toLowerCase();
+
+ if (
+ normalizedMimeType === MimeTypeAudio.WAV ||
+ normalizedMimeType === MimeTypeAudio.WAVE ||
+ normalizedMimeType === MimeTypeAudio.X_WAV ||
+ normalizedMimeType === MimeTypeAudio.X_WAVE ||
+ normalizedMimeType === MimeTypeAudio.VND_WAVE ||
+ normalizedMimeType === MimeTypeAudio.X_PN_WAV
+ ) {
+ return FileTypeAudio.WAV;
+ }
+
+ return FileTypeAudio.MP3;
+}
export {
deriveAgenticSections,
buildAssistantRawOutput,
- parseToolResultWithImages,
+ parseToolResultWithMedia,
splitSearchSummaryList,
hasAgenticContent,
classifyToolResult,
// CSS utilities
export { remToPx } from './css';
+
+// Audio format helper (used by agentic store and chat service)
+export { getAudioInputFormat } from './audio-format';
The point of the harness is the _scaling curve_, not any single number.
-| Knob | Reads on |
-| --------------------------- | ---------------------------------------------------------------------------------------------------- |
-| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. |
-| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). |
-| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. |
-| `openCodeFence` | `hljs.highlightAuto` on partial code. |
+| Knob | Reads on |
+| --------------------------- | --------------------------------------------------------------------------------------------------- |
+| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. |
+| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). |
+| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. |
+| `openCodeFence` | `hljs.highlightAuto` on partial code. |
Deliberately no hard assertions: CI timing is noisy and the value here is the
before/after delta, not a gate.
//
// Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts
-import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic';
+import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic';
import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code';
import { computeLineDiff } from '$lib/utils/compute-line-diff';
import { preprocessLaTeX } from '$lib/utils/latex-protection';
// --- per-line result parsers ----------------------------------------------
-describe('parseToolResultWithImages', () => {
+describe('parseToolResultWithMedia', () => {
bench('1KB', () => {
- parseToolResultWithImages(SHELL_OUTPUT_1KB, []);
+ parseToolResultWithMedia(SHELL_OUTPUT_1KB, []);
});
bench('200KB', () => {
- parseToolResultWithImages(SHELL_OUTPUT_200KB, []);
+ parseToolResultWithMedia(SHELL_OUTPUT_200KB, []);
});
bench('2MB', () => {
- parseToolResultWithImages(SHELL_OUTPUT_2MB, []);
+ parseToolResultWithMedia(SHELL_OUTPUT_2MB, []);
});
});
overview / pkg / ggml / sources / llama.cpp / commitdiff