server : (webui) rename has_multimodal --> modalities (#13393)

* server : (webui) rename has_multimodal --> modalities

* allow converting SVG to PNG

* less complicated code

diff --git a/tools/server/public/index.html.gz b/tools/server/public/index.html.gz
index 260fdcfecaee831ff26866b3466ee2fc8fdbbd4c..d7363e13eb25ae845b984662a2b8f42ee631f3d5 100644 (file)
Binary files a/tools/server/public/index.html.gz and b/tools/server/public/index.html.gz differ

diff --git a/tools/server/webui/src/components/useChatExtraContext.tsx b/tools/server/webui/src/components/useChatExtraContext.tsx
index 866401db9a7f23eab44beea94ebfbbdf5953fb9a..7eeff61f5e088b2780ef15056b9d964dde21f697 100644 (file)
--- a/tools/server/webui/src/components/useChatExtraContext.tsx
+++ b/tools/server/webui/src/components/useChatExtraContext.tsx
@@ -37,19 +37,26 @@ export function useChatExtraContext(): ChatExtraContextApi {
         break;
       }
 
-      if (mimeType.startsWith('image/') && mimeType !== 'image/svg+xml') {
-        if (!serverProps?.has_multimodal) {
+      if (mimeType.startsWith('image/')) {
+        if (!serverProps?.modalities?.vision) {
           toast.error('Multimodal is not supported by this server or model.');
           break;
         }
         const reader = new FileReader();
-        reader.onload = (event) => {
+        reader.onload = async (event) => {
           if (event.target?.result) {
+            let base64Url = event.target.result as string;
+
+            if (mimeType === 'image/svg+xml') {
+              // Convert SVG to PNG
+              base64Url = await svgBase64UrlToPngDataURL(base64Url);
+            }
+
             addItems([
               {
                 type: 'imageFile',
                 name: file.name,
-                base64Url: event.target.result as string,
+                base64Url,
               },
             ]);
           }
@@ -172,3 +179,56 @@ export function isLikelyNotBinary(str: string): boolean {
   const ratio = suspiciousCharCount / sampleLength;
   return ratio <= options.suspiciousCharThresholdRatio;
 }
+
+// WARN: vibe code below
+// Converts a Base64URL encoded SVG string to a PNG Data URL using browser Canvas API.
+function svgBase64UrlToPngDataURL(base64UrlSvg: string): Promise<string> {
+  const backgroundColor = 'white'; // Default background color for PNG
+
+  return new Promise((resolve, reject) => {
+    try {
+      const img = new Image();
+
+      img.onload = () => {
+        const canvas = document.createElement('canvas');
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          reject(new Error('Failed to get 2D canvas context.'));
+          return;
+        }
+
+        // Use provided dimensions or SVG's natural dimensions, with fallbacks
+        // Fallbacks (e.g., 300x300) are for SVGs without explicit width/height
+        // or when naturalWidth/Height might be 0 before full processing.
+        const targetWidth = img.naturalWidth || 300;
+        const targetHeight = img.naturalHeight || 300;
+
+        canvas.width = targetWidth;
+        canvas.height = targetHeight;
+
+        if (backgroundColor) {
+          ctx.fillStyle = backgroundColor;
+          ctx.fillRect(0, 0, canvas.width, canvas.height);
+        }
+
+        ctx.drawImage(img, 0, 0, targetWidth, targetHeight);
+        resolve(canvas.toDataURL('image/png'));
+      };
+
+      img.onerror = () => {
+        reject(
+          new Error('Failed to load SVG image. Ensure the SVG data is valid.')
+        );
+      };
+
+      // Load SVG string into an Image element
+      img.src = base64UrlSvg;
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const errorMessage = `Error converting SVG to PNG: ${message}`;
+      toast.error(errorMessage);
+      reject(new Error(errorMessage));
+    }
+  });
+}

diff --git a/tools/server/webui/src/utils/types.ts b/tools/server/webui/src/utils/types.ts
index add48be4cd2c12d814e326419678c24890040cd8..ba673dd9432da4aaa76201d2234d8140a7fa9cc5 100644 (file)
--- a/tools/server/webui/src/utils/types.ts
+++ b/tools/server/webui/src/utils/types.ts
@@ -118,6 +118,8 @@ export interface LlamaCppServerProps {
   build_info: string;
   model_path: string;
   n_ctx: number;
-  has_multimodal: boolean;
+  modalities?: {
+    vision: boolean;
+  };
   // TODO: support params
 }