From: Pascal Date: Sun, 26 Jul 2026 21:32:58 +0000 (+0200) Subject: ui: detect the conversation import format from file contents (#26121) X-Git-Tag: upstream/0.0.10438~300 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=55b7d6c4c7e518005bc320cf0139264411deac7f;p=pkg%2Fggml%2Fsources%2Fllama.cpp ui: detect the conversation import format from file contents (#26121) * ui: detect the conversation import format from file contents iOS resolves every accept entry to a UTI and has none for ".jsonl", so the picker greyed out exported conversations. Drop the accept filter and pick the parser from the file contents: ZIP magic bytes, then a first "session" record for JSONL, otherwise the legacy JSON format. Also remove the unused importConversations() picker and an orphan doc comment, and cover each format with unit tests. * ui: report what a conversation import actually wrote The import summary echoed the selection back, so re-importing conversations already in the database claimed success while nothing was written and only a console warning said otherwise. Return the imported and skipped conversations from the database layer, list the written ones in the summary, and count the rest in a toast. * ui: name the literals of the JSONL conversation format Introduce SessionRecordType and SESSION_HARNESS, and reuse the existing NEWLINE constant, so the record format lives in one place. This also covers the writer side, which predates the import path under review and carried the same literals: an enum stated by the reader alone lets the two sides drift. Values are unchanged, so an export stays byte identical. --- diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte index a86d68584..57dbba30b 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte @@ -159,8 +159,10 @@ try { const input = document.createElement('input'); + // No `accept` filter: iOS resolves each entry to a UTI and has none for + // `.jsonl`, which greys out exported conversations in the file picker. + // `parseImportFile` detects the format from the file contents instead. input.type = HtmlInputType.FILE; - input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`; input.onchange = async (e) => { const file = (e.target as HTMLInputElement)?.files?.[0]; @@ -199,9 +201,17 @@ .snapshot(fullImportData) .filter((item) => selectedIds.has(item.conv.id)); - await conversationsStore.importConversationsData(selectedData); + const { imported, skipped } = await conversationsStore.importConversationsData(selectedData); - importedConversations = selectedConversations; + // A conversation already in the database is left untouched, so the summary + // lists what was written and the toast accounts for the rest. + if (skipped.length > 0) { + toast.info( + `Skipped ${skipped.length} conversation${skipped.length === 1 ? '' : 's'} already in your library` + ); + } + + importedConversations = imported; showImportSummary = true; showExportSummary = false; showImportDialog = false; diff --git a/tools/ui/src/lib/constants/conversation-import.ts b/tools/ui/src/lib/constants/conversation-import.ts new file mode 100644 index 000000000..ed500440a --- /dev/null +++ b/tools/ui/src/lib/constants/conversation-import.ts @@ -0,0 +1,3 @@ +// First bytes of every ZIP local file header ("PK"). Import detects an archive +// from these bytes rather than from the filename, which the OS may not preserve. +export const ZIP_MAGIC = [0x50, 0x4b]; diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index a80e0cb63..100432c18 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -13,6 +13,7 @@ export * from './storage'; export * from './attachment-menu'; export * from './auto-scroll'; export * from './context-gauge-popup'; +export * from './conversation-import'; export * from './binary-detection'; export * from './built-in-tools'; export * from './cache'; diff --git a/tools/ui/src/lib/constants/message-export.ts b/tools/ui/src/lib/constants/message-export.ts index 79fa36f91..fc4dbe259 100644 --- a/tools/ui/src/lib/constants/message-export.ts +++ b/tools/ui/src/lib/constants/message-export.ts @@ -7,6 +7,9 @@ export const EXPORT_CONV_NAME_SUFFIX_MAX_LENGTH = 20; // Characters to keep in the ISO timestamp. 19 keeps 2026-01-01T00:00:00 export const ISO_TIMESTAMP_SLICE_LENGTH = 19; +// Producer marker carried by the session record of a JSONL export +export const SESSION_HARNESS = 'llama.app'; + // Replacements for making the conversation title filename-friendly export const NON_ALPHANUMERIC_REGEX = /[^a-z0-9]/gi; export const EXPORT_CONV_NONALNUM_REPLACEMENT = '_'; diff --git a/tools/ui/src/lib/enums/conversation-import.enums.ts b/tools/ui/src/lib/enums/conversation-import.enums.ts new file mode 100644 index 000000000..eef47c5cc --- /dev/null +++ b/tools/ui/src/lib/enums/conversation-import.enums.ts @@ -0,0 +1,9 @@ +/** + * Discriminator of a record line in the JSONL conversation format. A session + * record opens a conversation and carries its properties; every following + * message record belongs to it. + */ +export enum SessionRecordType { + SESSION = 'session', + MESSAGE = 'message' +} diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.ts index 2f70e063d..ee14293fc 100644 --- a/tools/ui/src/lib/enums/index.ts +++ b/tools/ui/src/lib/enums/index.ts @@ -27,6 +27,8 @@ export { ReasoningFormat } from './chat.enums'; +export { SessionRecordType } from './conversation-import.enums'; + export { ReasoningEffort } from './reasoning-effort.enums'; export { diff --git a/tools/ui/src/lib/services/database.service.ts b/tools/ui/src/lib/services/database.service.ts index 4fb70e29a..bc65caaca 100644 --- a/tools/ui/src/lib/services/database.service.ts +++ b/tools/ui/src/lib/services/database.service.ts @@ -554,12 +554,13 @@ export class DatabaseService { * Skips conversations that already exist. * * @param data - Array of { conv, messages } objects + * @returns The conversations written to the database and the ones skipped */ static async importConversations( data: { conv: DatabaseConversation; messages: DatabaseMessage[] }[] - ): Promise<{ imported: number; skipped: number }> { - let importedCount = 0; - let skippedCount = 0; + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { + const imported: DatabaseConversation[] = []; + const skipped: DatabaseConversation[] = []; return await db.transaction( 'rw', @@ -570,8 +571,7 @@ export class DatabaseService { const existing = await db[IDXDB_TABLES.conversations].get(conv.id); if (existing) { - console.warn(`Conversation "${conv.name}" already exists, skipping...`); - skippedCount++; + skipped.push(conv); continue; } @@ -580,10 +580,10 @@ export class DatabaseService { await db[IDXDB_TABLES.messages].put(msg); } - importedCount++; + imported.push(conv); } - return { imported: importedCount, skipped: skippedCount }; + return { imported, skipped }; } ); } diff --git a/tools/ui/src/lib/stores/conversations.svelte.ts b/tools/ui/src/lib/stores/conversations.svelte.ts index bc2feefd3..e467c8fad 100644 --- a/tools/ui/src/lib/stores/conversations.svelte.ts +++ b/tools/ui/src/lib/stores/conversations.svelte.ts @@ -30,11 +30,11 @@ import type { McpServerOverride } from '$lib/types/database'; import { zipSync, unzipSync, strToU8, strFromU8 } from 'fflate'; import { MessageRole, - HtmlInputType, FileExtensionText, MimeTypeText, MimeTypeApplication, - ReasoningEffort + ReasoningEffort, + SessionRecordType } from '$lib/enums'; import { ISO_DATE_TIME_SEPARATOR, @@ -47,7 +47,10 @@ import { ISO_TIME_SEPARATOR_REPLACEMENT, NON_ALPHANUMERIC_REGEX, MULTIPLE_UNDERSCORE_REGEX, - REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY + REASONING_EFFORT_DEFAULT_LOCALSTORAGE_KEY, + NEWLINE, + SESSION_HARNESS, + ZIP_MAGIC } from '$lib/constants'; import { ROUTES } from '$lib/constants/routes'; @@ -914,30 +917,35 @@ class ConversationsStore { /** * Serializes a session (a conversation with its messages) as JSONL. - * The first line is the session header (a `type: 'session'` record carrying the - * conversation properties); each subsequent line is a single message. + * The first line is the session header (a `SessionRecordType.SESSION` record + * carrying the conversation properties); each subsequent line is a single message. * @param data - The exported conversation payload * @returns The JSONL string (one record per line) */ serializeSessionToJsonl(data: ExportedConversation): string { const { conv, messages } = data; - const sessionLine = JSON.stringify({ type: 'session', harness: 'llama.app', ...conv }); + const sessionLine = JSON.stringify({ + type: SessionRecordType.SESSION, + harness: SESSION_HARNESS, + ...conv + }); const messageLines = messages.map((message: DatabaseMessage) => { // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. const { toolCalls, ...rest } = message; const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; - return JSON.stringify({ type: 'message', message: normalized }); + return JSON.stringify({ type: SessionRecordType.MESSAGE, message: normalized }); }); - return [sessionLine, ...messageLines].join('\n'); + return [sessionLine, ...messageLines].join(NEWLINE); } /** * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. - * A `type: 'session'` line starts a new session; following `type: 'message'` - * lines are appended to it. Supports multiple sessions in a single file. + * A `SessionRecordType.SESSION` line starts a new session; following + * `SessionRecordType.MESSAGE` lines are appended to it. Supports multiple + * sessions in a single file. * @param text - The JSONL file contents * @returns The parsed conversations with their messages */ @@ -945,20 +953,20 @@ class ConversationsStore { const sessions: ExportedConversation[] = []; let current: ExportedConversation | null = null; - for (const line of text.split('\n')) { + for (const line of text.split(NEWLINE)) { const trimmed = line.trim(); if (!trimmed) continue; const record = JSON.parse(trimmed); - if (record.type === 'session') { + if (record.type === SessionRecordType.SESSION) { // Drop the discriminator and harness marker; the rest is the conversation. const conv = { ...record }; delete conv.type; delete conv.harness; current = { conv: conv as DatabaseConversation, messages: [] }; sessions.push(current); - } else if (record.type === 'message') { + } else if (record.type === SessionRecordType.MESSAGE) { if (!current) { throw new Error('Invalid JSONL: message record before any session record'); } @@ -977,27 +985,47 @@ class ConversationsStore { } /** - * Parses an import file into conversations, accepting the current `.jsonl` and - * `.zip` formats as well as the legacy `.json` format. + * Reports whether the text is the JSONL session format, whose first non-empty + * line is a `SessionRecordType.SESSION` record. A legacy JSON export starts + * with an array or an object that has no such discriminator. + * @param text - The file contents + */ + private isSessionsJsonl(text: string): boolean { + const trimmed = text.trimStart(); + const lineEnd = trimmed.indexOf(NEWLINE); + const firstLine = lineEnd === -1 ? trimmed : trimmed.slice(0, lineEnd); + + try { + return JSON.parse(firstLine).type === SessionRecordType.SESSION; + } catch { + // Not a standalone JSON record, so not the JSONL format. + return false; + } + } + + /** + * Parses an import file into conversations, accepting the current JSONL and + * ZIP formats as well as the legacy JSON format. The format comes from the + * contents, so an import works whatever the file is named. * @param file - The user-selected file * @returns The parsed conversations with their messages */ async parseImportFile(file: File): Promise { - const name = file.name.toLowerCase(); + const bytes = new Uint8Array(await file.arrayBuffer()); - if (name.endsWith(FileExtensionText.ZIP)) { - const entries = unzipSync(new Uint8Array(await file.arrayBuffer())); + if (ZIP_MAGIC.every((byte, index) => bytes[index] === byte)) { + const entries = unzipSync(bytes); const sessions: ExportedConversation[] = []; - for (const [entryName, bytes] of Object.entries(entries)) { + for (const [entryName, entryBytes] of Object.entries(entries)) { if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; - sessions.push(...this.parseSessionsJsonl(strFromU8(bytes))); + sessions.push(...this.parseSessionsJsonl(strFromU8(entryBytes))); } return sessions; } - const text = await file.text(); + const text = strFromU8(bytes); - if (name.endsWith(FileExtensionText.JSONL)) { + if (this.isSessionsJsonl(text)) { return this.parseSessionsJsonl(text); } @@ -1103,73 +1131,14 @@ class ConversationsStore { this.downloadConversationFile({ conv: conversation, messages }); } - /** - * Imports conversations from a JSON file - * Opens file picker and processes the selected file - * @returns The list of imported conversations - */ - async importConversations(): Promise { - return new Promise((resolve, reject) => { - const input = document.createElement('input'); - input.type = HtmlInputType.FILE; - input.accept = FileExtensionText.JSON; - - input.onchange = async (e) => { - const file = (e.target as HTMLInputElement)?.files?.[0]; - - if (!file) { - reject(new Error('No file selected')); - return; - } - - try { - const text = await file.text(); - const parsedData = JSON.parse(text); - let importedData: ExportedConversations; - - if (Array.isArray(parsedData)) { - importedData = parsedData; - } else if ( - parsedData && - typeof parsedData === 'object' && - 'conv' in parsedData && - 'messages' in parsedData - ) { - importedData = [parsedData]; - } else { - throw new Error('Invalid file format'); - } - - const result = await DatabaseService.importConversations(importedData); - toast.success(`Imported ${result.imported} conversation(s), skipped ${result.skipped}`); - - await this.loadConversations(); - - const importedConversations = ( - Array.isArray(importedData) ? importedData : [importedData] - ).map((item) => item.conv); - - resolve(importedConversations); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Unknown error'; - console.error('Failed to import conversations:', err); - toast.error('Import failed', { description: message }); - reject(new Error(`Import failed: ${message}`)); - } - }; - - input.click(); - }); - } - /** * Imports conversations from provided data (without file picker) * @param data - Array of conversation data with messages - * @returns Import result with counts + * @returns The conversations written to the database and the ones skipped */ async importConversationsData( data: ExportedConversations - ): Promise<{ imported: number; skipped: number }> { + ): Promise<{ imported: DatabaseConversation[]; skipped: DatabaseConversation[] }> { const result = await DatabaseService.importConversations(data); await this.loadConversations(); return result; diff --git a/tools/ui/src/lib/utils/modality-file-validation.ts b/tools/ui/src/lib/utils/modality-file-validation.ts index bf78a7008..bfdee75ce 100644 --- a/tools/ui/src/lib/utils/modality-file-validation.ts +++ b/tools/ui/src/lib/utils/modality-file-validation.ts @@ -161,9 +161,3 @@ export function generateModalityErrorMessage( return message; } - -/** - * Generate file input accept string based on model modalities - * @param capabilities - The modality capabilities to check against - * @returns Accept string for HTML file input element - */ diff --git a/tools/ui/tests/client/conversation-import-db.svelte.test.ts b/tools/ui/tests/client/conversation-import-db.svelte.test.ts new file mode 100644 index 000000000..2a27be9b0 --- /dev/null +++ b/tools/ui/tests/client/conversation-import-db.svelte.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { DatabaseService } from '$lib/services/database.service'; +import { MessageRole, MessageType } from '$lib/enums'; +import type { ExportedConversation } from '$lib/types/database'; + +function makeSession(id: string): ExportedConversation { + return { + conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` }, + messages: [ + { + id: `${id}-msg`, + convId: id, + type: MessageType.TEXT, + timestamp: 0, + role: MessageRole.USER, + content: `hello from ${id}`, + parent: null, + children: [] + } + ] + } as unknown as ExportedConversation; +} + +afterEach(async () => { + const conversations = await DatabaseService.getAllConversations(); + await DatabaseService.bulkDeleteConversations(conversations.map((conv) => conv.id)); +}); + +/** + * An import leaves a conversation already in the database untouched, so the + * caller needs to know what was written to report it instead of echoing the + * selection back at the user. + */ +describe('DatabaseService.importConversations', () => { + it('reports the conversations it wrote', async () => { + const { imported, skipped } = await DatabaseService.importConversations([ + makeSession('a'), + makeSession('b') + ]); + + expect(imported.map((conv) => conv.id)).toEqual(['a', 'b']); + expect(skipped).toEqual([]); + expect(await DatabaseService.getConversationMessages('a')).toHaveLength(1); + }); + + it('reports an existing conversation as skipped and leaves it untouched', async () => { + await DatabaseService.importConversations([makeSession('a')]); + await DatabaseService.updateConversation('a', { name: 'Renamed locally' }); + + const { imported, skipped } = await DatabaseService.importConversations([makeSession('a')]); + + expect(imported).toEqual([]); + expect(skipped.map((conv) => conv.id)).toEqual(['a']); + expect((await DatabaseService.getConversation('a'))?.name).toBe('Renamed locally'); + }); + + it('imports the new conversations of a partially known selection', async () => { + await DatabaseService.importConversations([makeSession('a')]); + + const { imported, skipped } = await DatabaseService.importConversations([ + makeSession('a'), + makeSession('b') + ]); + + expect(imported.map((conv) => conv.id)).toEqual(['b']); + expect(skipped.map((conv) => conv.id)).toEqual(['a']); + }); +}); diff --git a/tools/ui/tests/unit/conversation-import.test.ts b/tools/ui/tests/unit/conversation-import.test.ts new file mode 100644 index 000000000..4565ed5b5 --- /dev/null +++ b/tools/ui/tests/unit/conversation-import.test.ts @@ -0,0 +1,112 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { zipSync, strToU8 } from 'fflate'; +import { MessageRole, MessageType } from '$lib/enums'; +import { NEWLINE } from '$lib/constants'; +import type { ExportedConversation } from '$lib/types/database'; + +let conversationsStore: typeof import('$lib/stores/conversations.svelte').conversationsStore; + +// node env unit project has no DOM, install a minimal localStorage backed by a +// Map before the store module reads it. Transforming the store takes seconds, +// so import it once for the whole file. +beforeAll(async () => { + const store = new Map(); + const polyfill: Storage = { + get length() { + return store.size; + }, + clear: () => store.clear(), + getItem: (k) => (store.has(k) ? store.get(k)! : null), + key: (i) => Array.from(store.keys())[i] ?? null, + removeItem: (k) => { + store.delete(k); + }, + setItem: (k, v) => { + store.set(k, String(v)); + } + }; + (globalThis as unknown as { localStorage: Storage }).localStorage = polyfill; + + ({ conversationsStore } = await import('$lib/stores/conversations.svelte')); +}, 30000); + +function makeSession(id: string): ExportedConversation { + return { + conv: { id, currNode: `${id}-msg`, lastModified: 0, name: `Chat ${id}` }, + messages: [ + { + id: `${id}-msg`, + convId: id, + type: MessageType.TEXT, + timestamp: 0, + role: MessageRole.USER, + content: `hello from ${id}`, + parent: null, + children: [] + } + ] + } as unknown as ExportedConversation; +} + +/** + * `parseImportFile` detects the format from the file contents. iOS has no UTI + * for `.jsonl`, so the picker cannot filter on it and the filename carries no + * guarantee: a JSONL export must import under any name. + */ +describe('conversationsStore.parseImportFile', () => { + it('imports a JSONL export whose name has no meaningful extension', async () => { + const jsonl = conversationsStore.serializeSessionToJsonl(makeSession('a')); + + const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + expect(sessions[0].messages[0].content).toBe('hello from a'); + }); + + it('imports several sessions from one JSONL file', async () => { + const jsonl = [makeSession('a'), makeSession('b')] + .map((session) => conversationsStore.serializeSessionToJsonl(session)) + .join(NEWLINE); + + const sessions = await conversationsStore.parseImportFile(new File([jsonl], 'export.txt')); + + expect(sessions.map((session) => session.conv.id)).toEqual(['a', 'b']); + }); + + it('imports a ZIP archive whose name has no meaningful extension', async () => { + const zipped = zipSync({ + 'a.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('a'))), + 'b.jsonl': strToU8(conversationsStore.serializeSessionToJsonl(makeSession('b'))), + 'notes.txt': strToU8('ignored') + }); + + const sessions = await conversationsStore.parseImportFile(new File([zipped], 'archive')); + + expect(sessions.map((session) => session.conv.id).sort()).toEqual(['a', 'b']); + }); + + it('imports the legacy JSON array format', async () => { + const json = JSON.stringify([makeSession('a')], null, 2); + + const sessions = await conversationsStore.parseImportFile(new File([json], 'export.jsonl')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + }); + + it('imports the legacy JSON single object format', async () => { + const json = JSON.stringify(makeSession('a')); + + const sessions = await conversationsStore.parseImportFile(new File([json], 'export')); + + expect(sessions).toHaveLength(1); + expect(sessions[0].conv.id).toBe('a'); + }); + + it('rejects a file that holds neither format', async () => { + await expect( + conversationsStore.parseImportFile(new File(['not an export'], 'export.jsonl')) + ).rejects.toThrow(); + }); +});