]> git.djapps.eu Git - pkg/ggml/sources/llama.cpp/commitdiff
ui: add opt-in run_javascript frontend tool (#24244)
authorPascal <redacted>
Tue, 9 Jun 2026 16:02:31 +0000 (18:02 +0200)
committerGitHub <redacted>
Tue, 9 Jun 2026 16:02:31 +0000 (18:02 +0200)
* ui: add opt-in run_javascript frontend tool

Expose a run_javascript tool to the model, executed entirely in the
browser through the existing agentic loop. Code runs in a Web Worker
inside a sandboxed iframe with an opaque origin, isolated from the
WebUI and its API. Console output, errors and the return value are
fed back as the tool result. The parent enforces a hard timeout by
removing the iframe, which terminates the worker.

Disabled by default, toggle in Settings > Developer.

* ui: address review feedback from allozaur

Use the JsonSchemaType enum for the tool definition parameter types
instead of raw string literals, extending it with STRING and NUMBER.

Move the worker shim and the iframe harness html into their own files
so the service no longer carries inline source blobs.

Replace the remaining magic strings with constants: SANDBOX_EMPTY_OUTPUT
and SANDBOX_TRUNCATION_NOTICE, and reuse NEWLINE_SEPARATOR for joins.

* ui: move sandbox worker shim to a raw imported file

Replace the inline worker template string with a real sandbox-worker.js
imported as raw text, and build the iframe harness from it in
sandbox-harness.ts. The raw worker ships as a string, not a module, so
it is excluded from eslint and the typecheck program.

15 files changed:
tools/ui/eslint.config.js
tools/ui/src/lib/constants/index.ts
tools/ui/src/lib/constants/sandbox.ts [new file with mode: 0644]
tools/ui/src/lib/constants/settings-keys.ts
tools/ui/src/lib/constants/settings-registry.ts
tools/ui/src/lib/constants/tools.ts
tools/ui/src/lib/enums/mcp.enums.ts
tools/ui/src/lib/enums/tools.enums.ts
tools/ui/src/lib/services/index.ts
tools/ui/src/lib/services/sandbox-harness.ts [new file with mode: 0644]
tools/ui/src/lib/services/sandbox-worker.js [new file with mode: 0644]
tools/ui/src/lib/services/sandbox.service.ts [new file with mode: 0644]
tools/ui/src/lib/stores/agentic.svelte.ts
tools/ui/src/lib/stores/tools.svelte.ts
tools/ui/tsconfig.json

index 6b3b1b5c04878c53b871fe3fe11743980b513bab..b90376b6c156163724d81ec433172ea8789f0a71 100644 (file)
@@ -46,7 +46,14 @@ export default ts.config(
        },
        {
                // Exclude generated build output and Storybook files from ESLint
-               ignores: ['dist/**', 'build/**', '.svelte-kit/**', 'test-results/**', '.storybook/**/*']
+               ignores: [
+                       'dist/**',
+                       'build/**',
+                       '.svelte-kit/**',
+                       'test-results/**',
+                       '.storybook/**/*',
+                       'src/lib/services/sandbox-worker.js'
+               ]
        },
        storybook.configs['flat/recommended']
 );
index c4334132c97ac3f4750b6658bd3164a5d57e5598..9ab864cd0e4ed64697245b85906a646c9ee6ac79 100644 (file)
@@ -37,6 +37,7 @@ export * from './model-id';
 export * from './precision';
 export * from './processing-info';
 export * from './routes';
+export * from './sandbox';
 export * from './settings-keys';
 export * from './settings-registry';
 export * from './supported-file-types';
diff --git a/tools/ui/src/lib/constants/sandbox.ts b/tools/ui/src/lib/constants/sandbox.ts
new file mode 100644 (file)
index 0000000..30e7695
--- /dev/null
@@ -0,0 +1,39 @@
+import { JsonSchemaType, ToolCallType } from '$lib/enums';
+import type { OpenAIToolDefinition } from '$lib/types';
+
+export const SANDBOX_TOOL_NAME = 'run_javascript';
+
+export const SANDBOX_TIMEOUT_MS_DEFAULT = 10000;
+
+export const SANDBOX_TIMEOUT_MS_MAX = 30000;
+
+export const SANDBOX_OUTPUT_MAX_CHARS = 8192;
+
+export const SANDBOX_EMPTY_OUTPUT = '(no output)';
+
+export const SANDBOX_TRUNCATION_NOTICE = '[output truncated]';
+
+export const SANDBOX_TOOL_DEFINITION: OpenAIToolDefinition = {
+       type: ToolCallType.FUNCTION,
+       function: {
+               name: SANDBOX_TOOL_NAME,
+               description:
+                       'Execute JavaScript in a sandboxed browser worker (no DOM, no page access). ' +
+                       'Top level await is supported. Use console.log to print intermediate values; ' +
+                       'a top level return statement is captured as the result.',
+               parameters: {
+                       type: JsonSchemaType.OBJECT,
+                       properties: {
+                               code: {
+                                       type: JsonSchemaType.STRING,
+                                       description: 'JavaScript source to execute'
+                               },
+                               timeout_ms: {
+                                       type: JsonSchemaType.NUMBER,
+                                       description: `Execution timeout in milliseconds, default ${SANDBOX_TIMEOUT_MS_DEFAULT}, max ${SANDBOX_TIMEOUT_MS_MAX}`
+                               }
+                       },
+                       required: ['code']
+               }
+       }
+};
index 5fff9f94c2ca005519af5487816382d7c3504357..3e19166dfdd6e34958830c884a9ecfb6e57d22af 100644 (file)
@@ -69,6 +69,7 @@ export const SETTINGS_KEYS = {
        ENABLE_THINKING: 'enableThinking',
        SHOW_RAW_OUTPUT_SWITCH: 'showRawOutputSwitch',
        // PY_INTERPRETER_ENABLED: 'pyInterpreterEnabled',
+       JS_SANDBOX_ENABLED: 'jsSandboxEnabled',
        CUSTOM_JSON: 'customJson',
        CUSTOM_CSS: 'customCss'
 } as const;
index 9246b970335aa50115c7334760a9463f103efc83..910e5c2df046883bdc9e96ce0d4c1d21c2fb087c 100644 (file)
@@ -690,6 +690,14 @@ const SETTINGS_REGISTRY: Record<string, SettingsSectionEntry> = {
                                        paramType: SyncableParameterType.BOOLEAN
                                }
                        },
+                       {
+                               key: SETTINGS_KEYS.JS_SANDBOX_ENABLED,
+                               label: 'JavaScript sandbox tool',
+                               help: 'Expose a run_javascript tool to the model. Code runs in a Web Worker inside a sandboxed iframe with an opaque origin, isolated from the WebUI and its API, with a hard timeout.',
+                               defaultValue: false,
+                               type: SettingsFieldType.CHECKBOX,
+                               section: SETTINGS_SECTION_SLUGS.DEVELOPER
+                       },
                        {
                                key: SETTINGS_KEYS.CUSTOM_JSON,
                                label: 'Custom JSON',
index efc3476cd7928d11ad0b0d80df3e484e61d24770..4d9385f9f5ca12d0bf8726dc13f4915e552ae599 100644 (file)
@@ -2,10 +2,12 @@ import { ToolSource } from '$lib/enums/tools.enums';
 
 export const TOOL_GROUP_LABELS = {
        [ToolSource.BUILTIN]: 'Built-in',
-       [ToolSource.CUSTOM]: 'JSON Schema'
+       [ToolSource.CUSTOM]: 'JSON Schema',
+       [ToolSource.FRONTEND]: 'Browser'
 } as const;
 
 export const TOOL_SERVER_LABELS = {
        [ToolSource.BUILTIN]: 'Built-in Tools',
-       [ToolSource.CUSTOM]: 'Custom Tools'
+       [ToolSource.CUSTOM]: 'Custom Tools',
+       [ToolSource.FRONTEND]: 'Browser Tools'
 } as const;
index d2c27e1a0c327d19c039a82580413fd68e1e22b1..3d9a2070dc541b43e95aa67f0a31cfb67f481e4a 100644 (file)
@@ -54,7 +54,9 @@ export enum MCPContentType {
  * JSON Schema types used in MCP tool definitions
  */
 export enum JsonSchemaType {
-       OBJECT = 'object'
+       OBJECT = 'object',
+       STRING = 'string',
+       NUMBER = 'number'
 }
 
 /**
index 4b2cdab3200c614aff0fbec8dbe141b8d0fa85b6..120c1d471d64b01f916331499597d32948cc41f6 100644 (file)
@@ -1,7 +1,8 @@
 export enum ToolSource {
        BUILTIN = 'builtin',
        MCP = 'mcp',
-       CUSTOM = 'custom'
+       CUSTOM = 'custom',
+       FRONTEND = 'frontend'
 }
 
 export enum ToolPermissionDecision {
index 475e6419bb9d57c9418e1a3733cc0736f96d16c0..386f740b8f04a4f1d5bcb11a9edae7ebf0f08e85 100644 (file)
@@ -261,6 +261,26 @@ export { ParameterSyncService } from './parameter-sync.service';
  */
 export { MCPService } from './mcp.service';
 
+/**
+ * **SandboxService** - Frontend JavaScript execution in a browser sandbox
+ *
+ * Stateless executor for the run_javascript frontend tool. Model generated
+ * code runs in a Web Worker spawned inside a sandboxed iframe with an opaque
+ * origin: no access to the app origin, its storage or its API, and outgoing
+ * requests carry a null origin. The code never touches a main thread, so the
+ * parent enforces the timeout by removing the iframe, which terminates the
+ * worker at the browser level.
+ *
+ * **Architecture & Relationships:**
+ * - **SandboxService** (this class): Stateless sandbox execution
+ * - **toolsStore**: Exposes the tool definition when the sandbox is enabled
+ * - **agenticStore**: Dispatches ToolSource.FRONTEND calls here
+ *
+ * @see SANDBOX_TOOL_DEFINITION in constants/sandbox.ts - tool schema sent to the LLM
+ * @see agenticStore in stores/agentic.svelte.ts - tool dispatch
+ */
+export { SandboxService } from './sandbox.service';
+
 /**
  * **RouterService** — Dynamic route URL construction utility
  *
diff --git a/tools/ui/src/lib/services/sandbox-harness.ts b/tools/ui/src/lib/services/sandbox-harness.ts
new file mode 100644 (file)
index 0000000..27b05e2
--- /dev/null
@@ -0,0 +1,25 @@
+import WORKER_SHIM from './sandbox-worker.js?raw';
+
+/**
+ * Harness loaded as srcdoc into a sandboxed iframe (allow-scripts only).
+ * The opaque origin is the security boundary: no access to the app origin,
+ * its storage or its API. The harness spawns a worker so model code never
+ * runs on a main thread, which makes the parent timeout enforceable by
+ * removing the iframe.
+ */
+export const SANDBOX_HARNESS_HTML = `<!doctype html><script>
+const SHIM = ${JSON.stringify(WORKER_SHIM)};
+addEventListener('message', (event) => {
+       const respond = (payload) => parent.postMessage(payload, '*');
+       let worker;
+       try {
+               worker = new Worker(URL.createObjectURL(new Blob([SHIM], { type: 'text/javascript' })));
+       } catch (err) {
+               respond({ logs: [], result: null, error: 'Worker creation failed: ' + err });
+               return;
+       }
+       worker.onmessage = (msg) => respond(msg.data);
+       worker.onerror = (err) => respond({ logs: [], result: null, error: String(err.message || err) });
+       worker.postMessage({ code: event.data.code });
+});
+</script>`;
diff --git a/tools/ui/src/lib/services/sandbox-worker.js b/tools/ui/src/lib/services/sandbox-worker.js
new file mode 100644 (file)
index 0000000..689b921
--- /dev/null
@@ -0,0 +1,30 @@
+const logs = [];
+const fmt = (value) => {
+       if (typeof value === 'string') return value;
+       try {
+               return JSON.stringify(value);
+       } catch {
+               return String(value);
+       }
+};
+const capture =
+       (level, prefix) =>
+       (...args) => {
+               logs.push(prefix + args.map(fmt).join(' '));
+       };
+console.log = capture('log', '');
+console.info = capture('info', '');
+console.debug = capture('debug', '');
+console.warn = capture('warn', 'warn: ');
+console.error = capture('error', 'error: ');
+self.onmessage = async (event) => {
+       const reply = { logs, result: null, error: null };
+       try {
+               const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+               const value = await new AsyncFunction(event.data.code)();
+               if (value !== undefined) reply.result = fmt(value);
+       } catch (err) {
+               reply.error = err instanceof Error ? err.stack || err.message : String(err);
+       }
+       self.postMessage(reply);
+};
diff --git a/tools/ui/src/lib/services/sandbox.service.ts b/tools/ui/src/lib/services/sandbox.service.ts
new file mode 100644 (file)
index 0000000..ef0e6cf
--- /dev/null
@@ -0,0 +1,112 @@
+import {
+       NEWLINE_SEPARATOR,
+       SANDBOX_EMPTY_OUTPUT,
+       SANDBOX_OUTPUT_MAX_CHARS,
+       SANDBOX_TIMEOUT_MS_DEFAULT,
+       SANDBOX_TIMEOUT_MS_MAX,
+       SANDBOX_TOOL_NAME,
+       SANDBOX_TRUNCATION_NOTICE
+} from '$lib/constants';
+import { SANDBOX_HARNESS_HTML } from './sandbox-harness';
+import type { ToolExecutionResult } from '$lib/types';
+
+interface SandboxReply {
+       logs?: unknown;
+       result?: unknown;
+       error?: unknown;
+}
+
+function formatReply(reply: SandboxReply): ToolExecutionResult {
+       const lines: string[] = [];
+
+       if (Array.isArray(reply.logs)) {
+               for (const line of reply.logs) lines.push(String(line));
+       }
+
+       if (reply.error != null) {
+               lines.push(`Error: ${String(reply.error)}`);
+       } else if (reply.result != null) {
+               lines.push(`=> ${String(reply.result)}`);
+       }
+
+       let content = lines.join(NEWLINE_SEPARATOR);
+       if (!content) content = SANDBOX_EMPTY_OUTPUT;
+       if (content.length > SANDBOX_OUTPUT_MAX_CHARS) {
+               content = `${content.slice(0, SANDBOX_OUTPUT_MAX_CHARS)}${NEWLINE_SEPARATOR}${SANDBOX_TRUNCATION_NOTICE}`;
+       }
+
+       return { content, isError: reply.error != null };
+}
+
+export class SandboxService {
+       /**
+        * Execute a frontend sandbox tool call and return its output.
+        * One disposable iframe per execution, removed on completion,
+        * timeout or abort. Removing the iframe terminates the worker
+        * at the browser level, so runaway code cannot outlive it.
+        */
+       static executeTool(
+               toolName: string,
+               params: Record<string, unknown>,
+               signal?: AbortSignal
+       ): Promise<ToolExecutionResult> {
+               if (toolName !== SANDBOX_TOOL_NAME) {
+                       return Promise.resolve({ content: `Unknown frontend tool: ${toolName}`, isError: true });
+               }
+
+               const code = typeof params.code === 'string' ? params.code : '';
+               if (!code) {
+                       return Promise.resolve({ content: 'Missing required parameter: code', isError: true });
+               }
+
+               const requested = Number(params.timeout_ms);
+               const timeoutMs =
+                       Number.isFinite(requested) && requested > 0
+                               ? Math.min(requested, SANDBOX_TIMEOUT_MS_MAX)
+                               : SANDBOX_TIMEOUT_MS_DEFAULT;
+
+               return new Promise<ToolExecutionResult>((resolve, reject) => {
+                       const iframe = document.createElement('iframe');
+                       iframe.setAttribute('sandbox', 'allow-scripts');
+                       iframe.style.display = 'none';
+                       iframe.srcdoc = SANDBOX_HARNESS_HTML;
+
+                       let settled = false;
+
+                       const cleanup = () => {
+                               settled = true;
+                               clearTimeout(timer);
+                               window.removeEventListener('message', onMessage);
+                               signal?.removeEventListener('abort', onAbort);
+                               iframe.remove();
+                       };
+
+                       const finish = (result: ToolExecutionResult) => {
+                               if (settled) return;
+                               cleanup();
+                               resolve(result);
+                       };
+
+                       const onAbort = () => {
+                               if (settled) return;
+                               cleanup();
+                               reject(new DOMException('Sandbox execution aborted', 'AbortError'));
+                       };
+
+                       const onMessage = (event: MessageEvent) => {
+                               if (event.source !== iframe.contentWindow) return;
+                               finish(formatReply((event.data ?? {}) as SandboxReply));
+                       };
+
+                       const timer = setTimeout(
+                               () => finish({ content: `Execution timed out after ${timeoutMs} ms`, isError: true }),
+                               timeoutMs
+                       );
+
+                       window.addEventListener('message', onMessage);
+                       signal?.addEventListener('abort', onAbort);
+                       iframe.onload = () => iframe.contentWindow?.postMessage({ code }, '*');
+                       document.body.appendChild(iframe);
+               });
+       }
+}
index 947737d7c15323f61ea767e1ccf8c1f2c6a531c8..5579cc1e5a6456127236faa1c5a7b86bf42184dd 100644 (file)
@@ -29,6 +29,7 @@ import { permissionsStore } from '$lib/stores/permissions.svelte';
 import { ToolSource, ToolPermissionDecision } from '$lib/enums';
 import { SvelteMap } from 'svelte/reactivity';
 import { ToolsService } from '$lib/services/tools.service';
+import { SandboxService } from '$lib/services/sandbox.service';
 import { isAbortError } from '$lib/utils';
 import { DEFAULT_AGENTIC_CONFIG, NEWLINE_SEPARATOR } from '$lib/constants';
 import {
@@ -784,6 +785,13 @@ class AgenticStore {
 
                                                        result = executionResult.content;
 
+                                                       if (executionResult.isError) toolSuccess = false;
+                                               } else if (toolSource === ToolSource.FRONTEND) {
+                                                       const args = this.parseToolArguments(toolCall.function.arguments);
+                                                       const executionResult = await SandboxService.executeTool(toolName, args, signal);
+
+                                                       result = executionResult.content;
+
                                                        if (executionResult.isError) toolSuccess = false;
                                                } else {
                                                        const mcpCall: MCPToolCall = {
index 82e41f0bf5b6581eb43c208194072c4b45578527..9f0101a82e41b31b11e500fe2dd154a6908575e5 100644 (file)
@@ -5,6 +5,7 @@ import { HealthCheckStatus, JsonSchemaType, ToolCallType, ToolSource } from '$li
 import { config } from '$lib/stores/settings.svelte';
 import {
        DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY,
+       SANDBOX_TOOL_DEFINITION,
        TOOL_GROUP_LABELS,
        TOOL_SERVER_LABELS
 } from '$lib/constants';
@@ -18,6 +19,8 @@ function toolKey(source: ToolSource, name: string, serverId?: string): string {
                        return serverId ? `mcp-${serverId}:${name}` : `mcp:${name}`;
                case ToolSource.CUSTOM:
                        return `custom:${name}`;
+               case ToolSource.FRONTEND:
+                       return `frontend:${name}`;
                default:
                        return `builtin:${name}`;
        }
@@ -82,6 +85,10 @@ class ToolsStore {
                return mcpStore.getToolDefinitionsForLLM();
        }
 
+       get frontendTools(): OpenAIToolDefinition[] {
+               return config().jsSandboxEnabled ? [SANDBOX_TOOL_DEFINITION] : [];
+       }
+
        get customTools(): OpenAIToolDefinition[] {
                const raw = config().customJson;
                if (!raw || typeof raw !== 'string') return [];
@@ -156,6 +163,15 @@ class ToolsStore {
                        push({ source: ToolSource.BUILTIN, key: toolKey(ToolSource.BUILTIN, name), definition: def });
                }
 
+               for (const def of this.frontendTools) {
+                       const name = def.function.name;
+                       push({
+                               source: ToolSource.FRONTEND,
+                               key: toolKey(ToolSource.FRONTEND, name),
+                               definition: def
+                       });
+               }
+
                for (const { serverId, serverName, definition } of this.mcpEntries()) {
                        const name = definition.function.name;
                        push({
@@ -208,6 +224,8 @@ class ToolsStore {
                                return entry.serverName ?? '';
                        case ToolSource.CUSTOM:
                                return TOOL_GROUP_LABELS[ToolSource.CUSTOM];
+                       case ToolSource.FRONTEND:
+                               return TOOL_GROUP_LABELS[ToolSource.FRONTEND];
                        default:
                                return TOOL_GROUP_LABELS[ToolSource.BUILTIN];
                }
@@ -237,6 +255,7 @@ class ToolsStore {
                };
 
                for (const def of this._builtinTools) take(def);
+               for (const def of this.frontendTools) take(def);
                for (const def of mcpStore.getToolDefinitionsForLLM()) take(def);
                for (const def of this.customTools) take(def);
 
@@ -346,6 +365,7 @@ class ToolsStore {
                if (entry.serverName) return mcpStore.getServerDisplayName(entry.serverName);
                if (entry.source === ToolSource.BUILTIN) return TOOL_SERVER_LABELS[ToolSource.BUILTIN];
                if (entry.source === ToolSource.CUSTOM) return TOOL_SERVER_LABELS[ToolSource.CUSTOM];
+               if (entry.source === ToolSource.FRONTEND) return TOOL_SERVER_LABELS[ToolSource.FRONTEND];
                return '';
        }
 
index 7c585f4db279992ebbcd142dec12db3d77eef4e9..51f55971e9dcb471aed28dc2f98c520c4853b431 100644 (file)
@@ -24,7 +24,8 @@
                "tests/**/*.svelte",
                ".storybook/**/*.ts",
                ".storybook/**/*.svelte"
-       ]
+       ],
+       "exclude": ["src/lib/services/sandbox-worker.js"]
        // Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias
        // except $lib which is handled by https://svelte.dev/docs/kit/configuration#files
        //