--- /dev/null
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import logging
+import argparse
+import os
+import sys
+import numpy
+import enum
+from pathlib import Path
+from typing import Any, Optional, Tuple, Type
+import warnings
+
+import numpy as np
+from PySide6.QtWidgets import (
+ QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
+ QPushButton, QLabel, QLineEdit, QFileDialog, QTableWidget,
+ QTableWidgetItem, QComboBox, QMessageBox, QTabWidget,
+ QTextEdit, QFormLayout,
+ QHeaderView, QDialog, QDialogButtonBox
+)
+from PySide6.QtCore import Qt
+
+# Necessary to load the local gguf package
+if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent.parent / 'gguf-py').exists():
+ sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+import gguf
+from gguf import GGUFReader, GGUFWriter, GGUFValueType, ReaderField
+from gguf.constants import TokenType, RopeScalingType, PoolingType, GGMLQuantizationType
+
+logger = logging.getLogger("gguf-editor-gui")
+
+# Map of key names to enum types for automatic enum interpretation
+KEY_TO_ENUM_TYPE = {
+ gguf.Keys.Tokenizer.TOKEN_TYPE: TokenType,
+ gguf.Keys.Rope.SCALING_TYPE: RopeScalingType,
+ gguf.Keys.LLM.POOLING_TYPE: PoolingType,
+ gguf.Keys.General.FILE_TYPE: GGMLQuantizationType,
+}
+
+# Define the tokenizer keys that should be edited together
+TOKENIZER_LINKED_KEYS = [
+ gguf.Keys.Tokenizer.LIST,
+ gguf.Keys.Tokenizer.TOKEN_TYPE,
+ gguf.Keys.Tokenizer.SCORES
+]
+
+
+class TokenizerEditorDialog(QDialog):
+ def __init__(self, tokens, token_types, scores, parent=None):
+ super().__init__(parent)
+ self.setWindowTitle("Edit Tokenizer Data")
+ self.resize(900, 600)
+
+ self.tokens = tokens.copy() if tokens else []
+ self.token_types = token_types.copy() if token_types else []
+ self.scores = scores.copy() if scores else []
+
+ # Ensure all arrays have the same length
+ max_len = max(len(self.tokens), len(self.token_types), len(self.scores))
+ if len(self.tokens) < max_len:
+ self.tokens.extend([""] * (max_len - len(self.tokens)))
+ if len(self.token_types) < max_len:
+ self.token_types.extend([0] * (max_len - len(self.token_types)))
+ if len(self.scores) < max_len:
+ self.scores.extend([0.0] * (max_len - len(self.scores)))
+
+ layout = QVBoxLayout(self)
+
+ # Add filter controls
+ filter_layout = QHBoxLayout()
+ filter_layout.addWidget(QLabel("Filter:"))
+ self.filter_edit = QLineEdit()
+ self.filter_edit.setPlaceholderText("Type to filter tokens...")
+ self.filter_edit.textChanged.connect(self.apply_filter)
+ filter_layout.addWidget(self.filter_edit)
+
+ # Add page controls
+ self.page_size = 100 # Show 100 items per page
+ self.current_page = 0
+ self.total_pages = max(1, (len(self.tokens) + self.page_size - 1) // self.page_size)
+
+ self.page_label = QLabel(f"Page 1 of {self.total_pages}")
+ filter_layout.addWidget(self.page_label)
+
+ prev_page = QPushButton("Previous")
+ prev_page.clicked.connect(self.previous_page)
+ filter_layout.addWidget(prev_page)
+
+ next_page = QPushButton("Next")
+ next_page.clicked.connect(self.next_page)
+ filter_layout.addWidget(next_page)
+
+ layout.addLayout(filter_layout)
+
+ # Tokenizer data table
+ self.tokens_table = QTableWidget()
+ self.tokens_table.setColumnCount(4)
+ self.tokens_table.setHorizontalHeaderLabels(["Index", "Token", "Type", "Score"])
+ self.tokens_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents)
+ self.tokens_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
+ self.tokens_table.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents)
+ self.tokens_table.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents)
+
+ layout.addWidget(self.tokens_table)
+
+ # Controls
+ controls_layout = QHBoxLayout()
+
+ add_button = QPushButton("Add Token")
+ add_button.clicked.connect(self.add_token)
+ controls_layout.addWidget(add_button)
+
+ remove_button = QPushButton("Remove Selected")
+ remove_button.clicked.connect(self.remove_selected)
+ controls_layout.addWidget(remove_button)
+
+ controls_layout.addStretch()
+
+ layout.addLayout(controls_layout)
+
+ # Buttons
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(self.accept)
+ buttons.rejected.connect(self.reject)
+ layout.addWidget(buttons)
+
+ # Initialize the filtered values
+ self.filtered_indices = list(range(len(self.tokens)))
+
+ # Load data for the first page
+ self.load_page()
+
+ def apply_filter(self):
+ """Filter the tokens based on the search text."""
+ filter_text = self.filter_edit.text().lower()
+
+ if not filter_text:
+ # No filter, show all values
+ self.filtered_indices = list(range(len(self.tokens)))
+ else:
+ # Apply filter
+ self.filtered_indices = []
+ for i, token in enumerate(self.tokens):
+ if filter_text in str(token).lower():
+ self.filtered_indices.append(i)
+
+ # Reset to first page and reload
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+ self.current_page = 0
+ self.page_label.setText(f"Page 1 of {self.total_pages}")
+ self.load_page()
+
+ def previous_page(self):
+ """Go to the previous page of results."""
+ if self.current_page > 0:
+ self.current_page -= 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+ self.load_page()
+
+ def next_page(self):
+ """Go to the next page of results."""
+ if self.current_page < self.total_pages - 1:
+ self.current_page += 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+ self.load_page()
+
+ def load_page(self):
+ """Load the current page of tokenizer data."""
+ self.tokens_table.setRowCount(0) # Clear the table
+
+ # Calculate start and end indices for the current page
+ start_idx = self.current_page * self.page_size
+ end_idx = min(start_idx + self.page_size, len(self.filtered_indices))
+
+ # Pre-allocate rows for better performance
+ self.tokens_table.setRowCount(end_idx - start_idx)
+
+ for row, i in enumerate(range(start_idx, end_idx)):
+ orig_idx = self.filtered_indices[i]
+
+ # Index
+ index_item = QTableWidgetItem(str(orig_idx))
+ index_item.setData(Qt.ItemDataRole.UserRole, orig_idx) # Store original index
+ index_item.setFlags(index_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tokens_table.setItem(row, 0, index_item)
+
+ # Token
+ token_item = QTableWidgetItem(str(self.tokens[orig_idx]))
+ self.tokens_table.setItem(row, 1, token_item)
+
+ # Token Type
+ token_type = self.token_types[orig_idx] if orig_idx < len(self.token_types) else 0
+ try:
+ enum_val = TokenType(token_type)
+ display_text = f"{enum_val.name} ({token_type})"
+ except (ValueError, KeyError):
+ display_text = f"Unknown ({token_type})"
+
+ type_item = QTableWidgetItem(display_text)
+ type_item.setData(Qt.ItemDataRole.UserRole, token_type)
+
+ # Make type cell editable with a double-click handler
+ type_item.setFlags(type_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tokens_table.setItem(row, 2, type_item)
+
+ # Score
+ score = self.scores[orig_idx] if orig_idx < len(self.scores) else 0.0
+ score_item = QTableWidgetItem(str(score))
+ self.tokens_table.setItem(row, 3, score_item)
+
+ # Connect double-click handler for token type cells
+ self.tokens_table.cellDoubleClicked.connect(self.handle_cell_double_click)
+
+ def handle_cell_double_click(self, row, column):
+ """Handle double-click on a cell, specifically for token type editing."""
+ if column == 2: # Token Type column
+ orig_item = self.tokens_table.item(row, 0)
+ if orig_item:
+ orig_idx = orig_item.data(Qt.ItemDataRole.UserRole)
+ self.edit_token_type(row, orig_idx)
+
+ def edit_token_type(self, row, orig_idx):
+ """Edit a token type using a dialog with a dropdown of all enum options."""
+ current_value = self.token_types[orig_idx] if orig_idx < len(self.token_types) else 0
+
+ # Create a dialog with enum options
+ dialog = QDialog(self)
+ dialog.setWindowTitle("Select Token Type")
+ layout = QVBoxLayout(dialog)
+
+ combo = QComboBox()
+ for enum_val in TokenType:
+ combo.addItem(f"{enum_val.name} ({enum_val.value})", enum_val.value)
+
+ # Set current value
+ try:
+ if isinstance(current_value, int):
+ enum_val = TokenType(current_value)
+ combo.setCurrentText(f"{enum_val.name} ({current_value})")
+ except (ValueError, KeyError):
+ pass
+
+ layout.addWidget(combo)
+
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(dialog.accept)
+ buttons.rejected.connect(dialog.reject)
+ layout.addWidget(buttons)
+
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ # Get the selected value
+ new_value = combo.currentData()
+ enum_val = TokenType(new_value)
+ display_text = f"{enum_val.name} ({new_value})"
+
+ # Update the display
+ type_item = self.tokens_table.item(row, 2)
+ if type_item:
+ type_item.setText(display_text)
+ type_item.setData(Qt.ItemDataRole.UserRole, new_value)
+
+ # Update the actual value
+ self.token_types[orig_idx] = new_value
+
+ def add_token(self):
+ """Add a new token to the end of the list."""
+ # Add to the end of the arrays
+ self.tokens.append("")
+ self.token_types.append(0) # Default to normal token
+ self.scores.append(0.0)
+
+ orig_idx = len(self.tokens) - 1
+
+ # Add to filtered indices if it matches the current filter
+ filter_text = self.filter_edit.text().lower()
+ if not filter_text or filter_text in "":
+ self.filtered_indices.append(orig_idx)
+
+ # Update pagination
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+
+ # Go to the last page to show the new item
+ self.current_page = self.total_pages - 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+
+ # Reload the page
+ self.load_page()
+
+ def remove_selected(self):
+ """Remove selected tokens from all arrays."""
+ selected_rows = []
+ for item in self.tokens_table.selectedItems():
+ row = item.row()
+ if row not in selected_rows:
+ selected_rows.append(row)
+
+ if not selected_rows:
+ return
+
+ # Get original indices in descending order to avoid index shifting
+ orig_indices = []
+ for row in selected_rows:
+ orig_item = self.tokens_table.item(row, 0)
+ if orig_item:
+ orig_indices.append(orig_item.data(Qt.ItemDataRole.UserRole))
+ orig_indices.sort(reverse=True)
+
+ # Remove from all arrays
+ for idx in orig_indices:
+ if idx < len(self.tokens):
+ del self.tokens[idx]
+ if idx < len(self.token_types):
+ del self.token_types[idx]
+ if idx < len(self.scores):
+ del self.scores[idx]
+
+ # Rebuild filtered_indices
+ self.filtered_indices = []
+ filter_text = self.filter_edit.text().lower()
+
+ for i, token in enumerate(self.tokens):
+ if not filter_text or filter_text in str(token).lower():
+ self.filtered_indices.append(i)
+
+ # Update pagination
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+ self.current_page = min(self.current_page, self.total_pages - 1)
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+
+ # Reload the page
+ self.load_page()
+
+ def get_data(self):
+ """Return the edited tokenizer data."""
+ return self.tokens, self.token_types, self.scores
+
+
+class ArrayEditorDialog(QDialog):
+ def __init__(self, array_values, element_type, key=None, parent=None):
+ super().__init__(parent)
+ self.setWindowTitle("Edit Array Values")
+ self.resize(700, 500)
+
+ self.array_values = array_values
+ self.element_type = element_type
+ self.key = key
+
+ # Get enum type for this array if applicable
+ self.enum_type = None
+ if key in KEY_TO_ENUM_TYPE and element_type == GGUFValueType.INT32:
+ self.enum_type = KEY_TO_ENUM_TYPE[key]
+
+ layout = QVBoxLayout(self)
+
+ # Add enum type information if applicable
+ if self.enum_type is not None:
+ enum_info_layout = QHBoxLayout()
+ enum_label = QLabel(f"Editing {self.enum_type.__name__} values:")
+ enum_info_layout.addWidget(enum_label)
+
+ # Add a legend for the enum values
+ enum_values = ", ".join([f"{e.name}={e.value}" for e in self.enum_type])
+ enum_values_label = QLabel(f"Available values: {enum_values}")
+ enum_values_label.setWordWrap(True)
+ enum_info_layout.addWidget(enum_values_label, 1)
+
+ layout.addLayout(enum_info_layout)
+
+ # Add search/filter controls
+ filter_layout = QHBoxLayout()
+ filter_layout.addWidget(QLabel("Filter:"))
+ self.filter_edit = QLineEdit()
+ self.filter_edit.setPlaceholderText("Type to filter values...")
+ self.filter_edit.textChanged.connect(self.apply_filter)
+ filter_layout.addWidget(self.filter_edit)
+
+ # Add page controls for large arrays
+ self.page_size = 100 # Show 100 items per page
+ self.current_page = 0
+ self.total_pages = max(1, (len(array_values) + self.page_size - 1) // self.page_size)
+
+ self.page_label = QLabel(f"Page 1 of {self.total_pages}")
+ filter_layout.addWidget(self.page_label)
+
+ prev_page = QPushButton("Previous")
+ prev_page.clicked.connect(self.previous_page)
+ filter_layout.addWidget(prev_page)
+
+ next_page = QPushButton("Next")
+ next_page.clicked.connect(self.next_page)
+ filter_layout.addWidget(next_page)
+
+ layout.addLayout(filter_layout)
+
+ # Array items table
+ self.items_table = QTableWidget()
+
+ # Set up columns based on whether we have an enum type
+ if self.enum_type is not None:
+ self.items_table.setColumnCount(3)
+ self.items_table.setHorizontalHeaderLabels(["Index", "Value", "Actions"])
+ self.items_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents)
+ self.items_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
+ self.items_table.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents)
+ else:
+ self.items_table.setColumnCount(2)
+ self.items_table.setHorizontalHeaderLabels(["Index", "Value"])
+ self.items_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.ResizeToContents)
+ self.items_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.Stretch)
+
+ layout.addWidget(self.items_table)
+
+ # Controls
+ controls_layout = QHBoxLayout()
+
+ add_button = QPushButton("Add Item")
+ add_button.clicked.connect(self.add_item)
+ controls_layout.addWidget(add_button)
+
+ remove_button = QPushButton("Remove Selected")
+ remove_button.clicked.connect(self.remove_selected)
+ controls_layout.addWidget(remove_button)
+
+ # Add bulk edit button for enum arrays
+ if self.enum_type is not None:
+ bulk_edit_button = QPushButton("Bulk Edit Selected")
+ bulk_edit_button.clicked.connect(self.bulk_edit_selected)
+ controls_layout.addWidget(bulk_edit_button)
+
+ controls_layout.addStretch()
+
+ layout.addLayout(controls_layout)
+
+ # Buttons
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(self.accept)
+ buttons.rejected.connect(self.reject)
+ layout.addWidget(buttons)
+
+ # Initialize the filtered values
+ self.filtered_indices = list(range(len(self.array_values)))
+
+ # Load array values for the first page
+ self.load_page()
+
+ def apply_filter(self):
+ """Filter the array values based on the search text."""
+ filter_text = self.filter_edit.text().lower()
+
+ if not filter_text:
+ # No filter, show all values
+ self.filtered_indices = list(range(len(self.array_values)))
+ else:
+ # Apply filter
+ self.filtered_indices = []
+ for i, value in enumerate(self.array_values):
+ # For enum values, search in both name and value
+ if self.enum_type is not None and isinstance(value, int):
+ try:
+ enum_val = self.enum_type(value)
+ display_text = f"{enum_val.name} ({value})".lower()
+ if filter_text in display_text:
+ self.filtered_indices.append(i)
+ except (ValueError, KeyError):
+ # If not a valid enum value, just check the raw value
+ if filter_text in str(value).lower():
+ self.filtered_indices.append(i)
+ else:
+ # For non-enum values, just check the string representation
+ if filter_text in str(value).lower():
+ self.filtered_indices.append(i)
+
+ # Reset to first page and reload
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+ self.current_page = 0
+ self.page_label.setText(f"Page 1 of {self.total_pages}")
+ self.load_page()
+
+ def previous_page(self):
+ """Go to the previous page of results."""
+ if self.current_page > 0:
+ self.current_page -= 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+ self.load_page()
+
+ def next_page(self):
+ """Go to the next page of results."""
+ if self.current_page < self.total_pages - 1:
+ self.current_page += 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+ self.load_page()
+
+ def load_page(self):
+ """Load the current page of array values."""
+ self.items_table.setRowCount(0) # Clear the table
+
+ # Calculate start and end indices for the current page
+ start_idx = self.current_page * self.page_size
+ end_idx = min(start_idx + self.page_size, len(self.filtered_indices))
+
+ # Pre-allocate rows for better performance
+ self.items_table.setRowCount(end_idx - start_idx)
+
+ for row, i in enumerate(range(start_idx, end_idx)):
+ orig_idx = self.filtered_indices[i]
+ value = self.array_values[orig_idx]
+
+ # Index
+ index_item = QTableWidgetItem(str(orig_idx))
+ index_item.setData(Qt.ItemDataRole.UserRole, orig_idx) # Store original index
+ index_item.setFlags(index_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.items_table.setItem(row, 0, index_item)
+
+ # Value
+ if self.enum_type is not None:
+ # Display enum value and name
+ try:
+ if isinstance(value, (int, numpy.signedinteger)):
+ enum_val = self.enum_type(value)
+ display_text = f"{enum_val.name} ({value})"
+ else:
+ display_text = str(value)
+ except (ValueError, KeyError):
+ display_text = f"Unknown ({value})"
+
+ # Store the enum value in the item
+ value_item = QTableWidgetItem(display_text)
+ value_item.setData(Qt.ItemDataRole.UserRole, value)
+ value_item.setFlags(value_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.items_table.setItem(row, 1, value_item)
+
+ # Add an edit button in a separate column
+ edit_button = QPushButton("Edit")
+ edit_button.setProperty("row", row)
+ edit_button.clicked.connect(self.edit_array_enum_value)
+
+ # Create a widget to hold the button
+ button_widget = QWidget()
+ button_layout = QHBoxLayout(button_widget)
+ button_layout.setContentsMargins(2, 2, 2, 2)
+ button_layout.addWidget(edit_button)
+ button_layout.addStretch()
+
+ self.items_table.setCellWidget(row, 2, button_widget)
+ else:
+ value_item = QTableWidgetItem(str(value))
+ self.items_table.setItem(row, 1, value_item)
+
+ def edit_array_enum_value(self):
+ """Handle editing an enum value in the array editor."""
+ button = self.sender()
+ row = button.property("row")
+
+ # Get the original index from the table item
+ orig_item = self.items_table.item(row, 0)
+ new_item = self.items_table.item(row, 1)
+ if orig_item and new_item and self.enum_type and self.edit_enum_value(row, self.enum_type):
+ orig_idx = orig_item.data(Qt.ItemDataRole.UserRole)
+ new_value = new_item.data(Qt.ItemDataRole.UserRole)
+ # Update the stored value in the array
+ if isinstance(new_value, (int, float, str, bool)):
+ self.array_values[orig_idx] = new_value
+
+ def bulk_edit_selected(self):
+ """Edit multiple enum values at once."""
+ if not self.enum_type:
+ return
+
+ selected_rows = set()
+ for item in self.items_table.selectedItems():
+ selected_rows.add(item.row())
+
+ if not selected_rows:
+ QMessageBox.information(self, "No Selection", "Please select at least one row to edit.")
+ return
+
+ # Create a dialog with enum options
+ dialog = QDialog(self)
+ dialog.setWindowTitle(f"Bulk Edit {self.enum_type.__name__} Values")
+ layout = QVBoxLayout(dialog)
+
+ layout.addWidget(QLabel(f"Set {len(selected_rows)} selected items to:"))
+
+ combo = QComboBox()
+ for enum_val in self.enum_type:
+ combo.addItem(f"{enum_val.name} ({enum_val.value})", enum_val.value)
+
+ layout.addWidget(combo)
+
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(dialog.accept)
+ buttons.rejected.connect(dialog.reject)
+ layout.addWidget(buttons)
+
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ # Get the selected value
+ new_value = combo.currentData()
+ enum_val = self.enum_type(new_value)
+ display_text = f"{enum_val.name} ({new_value})"
+
+ # Update all selected rows
+ for row in selected_rows:
+ orig_item = self.items_table.item(row, 0)
+ new_item = self.items_table.item(row, 1)
+ if orig_item and new_item:
+ orig_idx = orig_item.data(Qt.ItemDataRole.UserRole)
+ self.array_values[orig_idx] = new_value
+
+ # Update the display
+ new_item.setText(display_text)
+ new_item.setData(Qt.ItemDataRole.UserRole, new_value)
+
+ def add_item(self):
+ # Add to the end of the array
+ orig_idx = len(self.array_values)
+
+ # Add default value based on type
+ if self.enum_type is not None:
+ # Default to first enum value
+ default_value = list(self.enum_type)[0].value
+ self.array_values.append(default_value)
+ else:
+ if self.element_type == GGUFValueType.STRING:
+ self.array_values.append("")
+ else:
+ self.array_values.append(0)
+
+ # Add to filtered indices if it matches the current filter
+ self.filtered_indices.append(orig_idx)
+
+ # Update pagination
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+
+ # Go to the last page to show the new item
+ self.current_page = self.total_pages - 1
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+
+ # Reload the page
+ self.load_page()
+
+ def remove_selected(self):
+ selected_rows = []
+ for item in self.items_table.selectedItems():
+ row = item.row()
+ if row not in selected_rows:
+ selected_rows.append(row)
+
+ if not selected_rows:
+ return
+
+ # Get original indices in descending order to avoid index shifting
+ orig_indices = list()
+ for row in selected_rows:
+ orig_item = self.items_table.item(row, 0)
+ if orig_item:
+ orig_indices.append(orig_item.data(Qt.ItemDataRole.UserRole))
+ orig_indices.sort(reverse=True)
+
+ # Remove from array_values
+ for idx in orig_indices:
+ del self.array_values[idx]
+
+ # Rebuild filtered_indices
+ self.filtered_indices = []
+ filter_text = self.filter_edit.text().lower()
+
+ for i, value in enumerate(self.array_values):
+ if not filter_text:
+ self.filtered_indices.append(i)
+ else:
+ # Apply filter
+ if self.enum_type is not None and isinstance(value, int):
+ try:
+ enum_val = self.enum_type(value)
+ display_text = f"{enum_val.name} ({value})".lower()
+ if filter_text in display_text:
+ self.filtered_indices.append(i)
+ except (ValueError, KeyError):
+ if filter_text in str(value).lower():
+ self.filtered_indices.append(i)
+ else:
+ if filter_text in str(value).lower():
+ self.filtered_indices.append(i)
+
+ # Update pagination
+ self.total_pages = max(1, (len(self.filtered_indices) + self.page_size - 1) // self.page_size)
+ self.current_page = min(self.current_page, self.total_pages - 1)
+ self.page_label.setText(f"Page {self.current_page + 1} of {self.total_pages}")
+
+ # Reload the page
+ self.load_page()
+
+ def edit_enum_value(self, row: int, enum_type: Type[enum.Enum]):
+ """Edit an enum value using a dialog with a dropdown of all enum options."""
+ # Get the original index from the table item
+ orig_item = self.items_table.item(row, 0)
+ if orig_item:
+ orig_idx = orig_item.data(Qt.ItemDataRole.UserRole)
+ else:
+ return
+ current_value = self.array_values[orig_idx]
+
+ # Create a dialog with enum options
+ dialog = QDialog(self)
+ dialog.setWindowTitle(f"Select {enum_type.__name__} Value")
+ layout = QVBoxLayout(dialog)
+
+ # Add description
+ description = QLabel(f"Select a {enum_type.__name__} value:")
+ layout.addWidget(description)
+
+ # Use a combo box for quick selection
+ combo = QComboBox()
+ for enum_val in enum_type:
+ combo.addItem(f"{enum_val.name} ({enum_val.value})", enum_val.value)
+
+ # Set current value
+ try:
+ if isinstance(current_value, int):
+ enum_val = enum_type(current_value)
+ combo.setCurrentText(f"{enum_val.name} ({current_value})")
+ except (ValueError, KeyError):
+ pass
+
+ layout.addWidget(combo)
+
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(dialog.accept)
+ buttons.rejected.connect(dialog.reject)
+ layout.addWidget(buttons)
+
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ # Update the value display and stored data
+ new_value = combo.currentData()
+ enum_val = enum_type(new_value)
+ display_text = f"{enum_val.name} ({new_value})"
+
+ new_item = self.items_table.item(row, 1)
+ if new_item:
+ new_item.setText(display_text)
+ new_item.setData(Qt.ItemDataRole.UserRole, new_value)
+
+ # Update the actual array value
+ self.array_values[orig_idx] = new_value
+ return True
+ return False
+
+ def get_array_values(self):
+ # The array_values list is kept up-to-date as edits are made
+ return self.array_values
+
+
+class AddMetadataDialog(QDialog):
+ def __init__(self, parent=None):
+ super().__init__(parent)
+ self.setWindowTitle("Add Metadata")
+ self.resize(400, 200)
+
+ layout = QVBoxLayout(self)
+
+ form_layout = QFormLayout()
+
+ self.key_edit = QLineEdit()
+ form_layout.addRow("Key:", self.key_edit)
+
+ self.type_combo = QComboBox()
+ for value_type in GGUFValueType:
+ if value_type != GGUFValueType.ARRAY: # Skip array type for simplicity
+ self.type_combo.addItem(value_type.name, value_type)
+ form_layout.addRow("Type:", self.type_combo)
+
+ self.value_edit = QTextEdit()
+ form_layout.addRow("Value:", self.value_edit)
+
+ layout.addLayout(form_layout)
+
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(self.accept)
+ buttons.rejected.connect(self.reject)
+ layout.addWidget(buttons)
+
+ def get_data(self) -> Tuple[str, GGUFValueType, Any]:
+ key = self.key_edit.text()
+ value_type = self.type_combo.currentData()
+ value_text = self.value_edit.toPlainText()
+
+ # Convert value based on type
+ if value_type == GGUFValueType.UINT8:
+ value = np.uint8(int(value_text))
+ elif value_type == GGUFValueType.INT8:
+ value = np.int8(int(value_text))
+ elif value_type == GGUFValueType.UINT16:
+ value = np.uint16(int(value_text))
+ elif value_type == GGUFValueType.INT16:
+ value = np.int16(int(value_text))
+ elif value_type == GGUFValueType.UINT32:
+ value = np.uint32(int(value_text))
+ elif value_type == GGUFValueType.INT32:
+ value = np.int32(int(value_text))
+ elif value_type == GGUFValueType.FLOAT32:
+ value = np.float32(float(value_text))
+ elif value_type == GGUFValueType.BOOL:
+ value = value_text.lower() in ('true', 'yes', '1')
+ elif value_type == GGUFValueType.STRING:
+ value = value_text
+ else:
+ value = value_text
+
+ return key, value_type, value
+
+
+class GGUFEditorWindow(QMainWindow):
+ def __init__(self):
+ super().__init__()
+
+ self.setWindowTitle("GGUF Editor")
+ self.resize(1000, 800)
+
+ self.current_file = None
+ self.reader = None
+ self.modified = False
+ self.metadata_changes = {} # Store changes to apply when saving
+ self.metadata_to_remove = set() # Store keys to remove when saving
+
+ self.setup_ui()
+
+ def setup_ui(self):
+ central_widget = QWidget()
+ self.setCentralWidget(central_widget)
+
+ main_layout = QVBoxLayout(central_widget)
+
+ # File controls
+ file_layout = QHBoxLayout()
+
+ self.file_path_edit = QLineEdit()
+ self.file_path_edit.setReadOnly(True)
+ file_layout.addWidget(self.file_path_edit)
+
+ open_button = QPushButton("Open GGUF")
+ open_button.clicked.connect(self.open_file)
+ file_layout.addWidget(open_button)
+
+ save_button = QPushButton("Save As...")
+ save_button.clicked.connect(self.save_file)
+ file_layout.addWidget(save_button)
+
+ main_layout.addLayout(file_layout)
+
+ # Tabs for different views
+ self.tabs = QTabWidget()
+
+ # Metadata tab
+ self.metadata_tab = QWidget()
+ metadata_layout = QVBoxLayout(self.metadata_tab)
+
+ # Metadata table
+ self.metadata_table = QTableWidget()
+ self.metadata_table.setColumnCount(4)
+ self.metadata_table.setHorizontalHeaderLabels(["Key", "Type", "Value", "Actions"])
+ self.metadata_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch)
+ self.metadata_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.ResizeToContents)
+ self.metadata_table.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.Stretch)
+ self.metadata_table.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents)
+ metadata_layout.addWidget(self.metadata_table)
+
+ # Metadata controls
+ metadata_controls = QHBoxLayout()
+
+ add_metadata_button = QPushButton("Add Metadata")
+ add_metadata_button.clicked.connect(self.add_metadata)
+ metadata_controls.addWidget(add_metadata_button)
+
+ metadata_controls.addStretch()
+
+ metadata_layout.addLayout(metadata_controls)
+
+ # Tensors tab
+ self.tensors_tab = QWidget()
+ tensors_layout = QVBoxLayout(self.tensors_tab)
+
+ self.tensors_table = QTableWidget()
+ self.tensors_table.setColumnCount(5)
+ self.tensors_table.setHorizontalHeaderLabels(["Name", "Type", "Shape", "Elements", "Size (bytes)"])
+ self.tensors_table.horizontalHeader().setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch)
+ self.tensors_table.horizontalHeader().setSectionResizeMode(1, QHeaderView.ResizeMode.ResizeToContents)
+ self.tensors_table.horizontalHeader().setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents)
+ self.tensors_table.horizontalHeader().setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents)
+ self.tensors_table.horizontalHeader().setSectionResizeMode(4, QHeaderView.ResizeMode.ResizeToContents)
+ tensors_layout.addWidget(self.tensors_table)
+
+ # Add tabs to tab widget
+ self.tabs.addTab(self.metadata_tab, "Metadata")
+ self.tabs.addTab(self.tensors_tab, "Tensors")
+
+ main_layout.addWidget(self.tabs)
+
+ # Status bar
+ self.statusBar().showMessage("Ready")
+
+ def load_file(self, file_path):
+ """Load a GGUF file by path"""
+ try:
+ self.statusBar().showMessage(f"Loading {file_path}...")
+ QApplication.processEvents()
+
+ self.reader = GGUFReader(file_path, 'r')
+ self.current_file = file_path
+ self.file_path_edit.setText(file_path)
+
+ self.load_metadata()
+ self.load_tensors()
+
+ self.metadata_changes = {}
+ self.metadata_to_remove = set()
+ self.modified = False
+
+ self.statusBar().showMessage(f"Loaded {file_path}")
+ return True
+ except Exception as e:
+ QMessageBox.critical(self, "Error", f"Failed to open file: {str(e)}")
+ self.statusBar().showMessage("Error loading file")
+ return False
+
+ def open_file(self):
+ file_path, _ = QFileDialog.getOpenFileName(
+ self, "Open GGUF File", "", "GGUF Files (*.gguf);;All Files (*)"
+ )
+
+ if not file_path:
+ return
+
+ self.load_file(file_path)
+
+ def load_metadata(self):
+ self.metadata_table.setRowCount(0)
+
+ if not self.reader:
+ return
+
+ # Disconnect to prevent triggering during loading
+ with warnings.catch_warnings():
+ warnings.filterwarnings('ignore')
+ self.metadata_table.itemChanged.disconnect(self.on_metadata_changed)
+
+ for i, (key, field) in enumerate(self.reader.fields.items()):
+ self.metadata_table.insertRow(i)
+
+ # Key
+ key_item = QTableWidgetItem(key)
+ key_item.setFlags(key_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.metadata_table.setItem(i, 0, key_item)
+
+ # Type
+ if not field.types:
+ type_str = "N/A"
+ elif field.types[0] == GGUFValueType.ARRAY:
+ nest_count = len(field.types) - 1
+ element_type = field.types[-1].name
+ # Check if this is an enum array
+ enum_type = self.get_enum_for_key(key)
+ if enum_type is not None and field.types[-1] == GGUFValueType.INT32:
+ element_type = enum_type.__name__
+ type_str = '[' * nest_count + element_type + ']' * nest_count
+ else:
+ type_str = str(field.types[0].name)
+ # Check if this is an enum field
+ enum_type = self.get_enum_for_key(key)
+ if enum_type is not None and field.types[0] == GGUFValueType.INT32:
+ type_str = enum_type.__name__
+
+ type_item = QTableWidgetItem(type_str)
+ type_item.setFlags(type_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.metadata_table.setItem(i, 1, type_item)
+
+ # Value
+ value_str = self.format_field_value(field)
+ value_item = QTableWidgetItem(value_str)
+
+ # Make only simple values editable
+ if len(field.types) == 1 and field.types[0] != GGUFValueType.ARRAY:
+ value_item.setFlags(value_item.flags() | Qt.ItemFlag.ItemIsEditable)
+ else:
+ value_item.setFlags(value_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+
+ self.metadata_table.setItem(i, 2, value_item)
+
+ # Actions
+ actions_widget = QWidget()
+ actions_layout = QHBoxLayout(actions_widget)
+ actions_layout.setContentsMargins(2, 2, 2, 2)
+
+ # Add Edit button for arrays and enum fields
+ if field.types and field.types[0] == GGUFValueType.ARRAY:
+ edit_button = QPushButton("Edit")
+ edit_button.setProperty("row", i)
+ edit_button.setProperty("key", key)
+ edit_button.clicked.connect(self.edit_array_metadata)
+ actions_layout.addWidget(edit_button)
+
+ # Add special label for tokenizer linked fields
+ if key in TOKENIZER_LINKED_KEYS:
+ edit_button.setText("Edit Tokenizer")
+ edit_button.setToolTip("Edit all tokenizer data together")
+ elif len(field.types) == 1 and self.get_enum_for_key(key) is not None:
+ edit_button = QPushButton("Edit")
+ edit_button.setProperty("row", i)
+ edit_button.setProperty("key", key)
+ edit_button.clicked.connect(self.edit_metadata_enum)
+ actions_layout.addWidget(edit_button)
+
+ remove_button = QPushButton("Remove")
+ remove_button.setProperty("row", i)
+ remove_button.setProperty("key", key)
+ remove_button.clicked.connect(self.remove_metadata)
+ actions_layout.addWidget(remove_button)
+
+ self.metadata_table.setCellWidget(i, 3, actions_widget)
+
+ # Reconnect after loading
+ self.metadata_table.itemChanged.connect(self.on_metadata_changed)
+
+ def extract_array_values(self, field: ReaderField) -> list:
+ """Extract all values from an array field."""
+ if not field.types or field.types[0] != GGUFValueType.ARRAY:
+ return []
+
+ curr_type = field.types[1]
+ array_values = []
+ total_elements = len(field.data)
+
+ if curr_type == GGUFValueType.STRING:
+ for element_pos in range(total_elements):
+ value_string = str(bytes(field.parts[-1 - (total_elements - element_pos - 1) * 2]), encoding='utf-8')
+ array_values.append(value_string)
+ elif self.reader and curr_type in self.reader.gguf_scalar_to_np:
+ for element_pos in range(total_elements):
+ array_values.append(field.parts[-1 - (total_elements - element_pos - 1)][0])
+
+ return array_values
+
+ def get_enum_for_key(self, key: str) -> Optional[Type[enum.Enum]]:
+ """Get the enum type for a given key if it exists."""
+ return KEY_TO_ENUM_TYPE.get(key)
+
+ def format_enum_value(self, value: Any, enum_type: Type[enum.Enum]) -> str:
+ """Format a value as an enum if possible."""
+ try:
+ if isinstance(value, (int, str)):
+ enum_value = enum_type(value)
+ return f"{enum_value.name} ({value})"
+ except (ValueError, KeyError):
+ pass
+ return str(value)
+
+ def format_field_value(self, field: ReaderField) -> str:
+ if not field.types:
+ return "N/A"
+
+ if len(field.types) == 1:
+ curr_type = field.types[0]
+ if curr_type == GGUFValueType.STRING:
+ return str(bytes(field.parts[-1]), encoding='utf-8')
+ elif self.reader and curr_type in self.reader.gguf_scalar_to_np:
+ value = field.parts[-1][0]
+ # Check if this field has an enum type
+ enum_type = self.get_enum_for_key(field.name)
+ if enum_type is not None:
+ return self.format_enum_value(value, enum_type)
+ return str(value)
+
+ if field.types[0] == GGUFValueType.ARRAY:
+ array_values = self.extract_array_values(field)
+ render_element = min(5, len(array_values))
+
+ # Get enum type for this array if applicable
+ enum_type = self.get_enum_for_key(field.name)
+
+ if enum_type is not None:
+ array_elements = []
+ for i in range(render_element):
+ array_elements.append(self.format_enum_value(array_values[i], enum_type))
+ else:
+ array_elements = [str(array_values[i]) for i in range(render_element)]
+
+ return f"[ {', '.join(array_elements).strip()}{', ...' if len(array_values) > len(array_elements) else ''} ]"
+
+ return "Complex value"
+
+ def load_tensors(self):
+ self.tensors_table.setRowCount(0)
+
+ if not self.reader:
+ return
+
+ for i, tensor in enumerate(self.reader.tensors):
+ self.tensors_table.insertRow(i)
+
+ # Name
+ name_item = QTableWidgetItem(tensor.name)
+ name_item.setFlags(name_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tensors_table.setItem(i, 0, name_item)
+
+ # Type
+ type_item = QTableWidgetItem(tensor.tensor_type.name)
+ type_item.setFlags(type_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tensors_table.setItem(i, 1, type_item)
+
+ # Shape
+ shape_str = " × ".join(str(d) for d in tensor.shape)
+ shape_item = QTableWidgetItem(shape_str)
+ shape_item.setFlags(shape_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tensors_table.setItem(i, 2, shape_item)
+
+ # Elements
+ elements_item = QTableWidgetItem(str(tensor.n_elements))
+ elements_item.setFlags(elements_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tensors_table.setItem(i, 3, elements_item)
+
+ # Size
+ size_item = QTableWidgetItem(f"{tensor.n_bytes:,}")
+ size_item.setFlags(size_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.tensors_table.setItem(i, 4, size_item)
+
+ def on_metadata_changed(self, item):
+ if item.column() != 2: # Only handle value column changes
+ return
+
+ row = item.row()
+ orig_item = self.metadata_table.item(row, 0)
+ key = None
+ if orig_item:
+ key = orig_item.text()
+ new_value = item.text()
+
+ field = None
+ if self.reader and key:
+ field = self.reader.get_field(key)
+ if not field or not field.types or not key:
+ return
+
+ value_type = field.types[0]
+
+ # Check if this is an enum field
+ enum_type = self.get_enum_for_key(key)
+ if enum_type is not None and value_type == GGUFValueType.INT32:
+ # Try to parse the enum value from the text
+ try:
+ # Check if it's a name
+ try:
+ enum_val = enum_type[new_value]
+ converted_value = enum_val.value
+ except (KeyError, AttributeError):
+ # Check if it's a number or "NAME (value)" format
+ if '(' in new_value and ')' in new_value:
+ # Extract the value from "NAME (value)" format
+ value_part = new_value.split('(')[1].split(')')[0].strip()
+ converted_value = int(value_part)
+ else:
+ # Try to convert directly to int
+ converted_value = int(new_value)
+
+ # Validate that it's a valid enum value
+ enum_type(converted_value)
+
+ # Store the change
+ self.metadata_changes[key] = (value_type, converted_value)
+ self.modified = True
+
+ # Update display with formatted enum value
+ formatted_value = self.format_enum_value(converted_value, enum_type)
+ item.setText(formatted_value)
+
+ self.statusBar().showMessage(f"Changed {key} to {formatted_value}")
+ return
+ except (ValueError, KeyError) as e:
+ QMessageBox.warning(
+ self,
+ f"Invalid Enum Value ({e})",
+ f"'{new_value}' is not a valid {enum_type.__name__} value.\n"
+ f"Valid values are: {', '.join(v.name for v in enum_type)}")
+
+ # Revert to original value
+ original_value = self.format_field_value(field)
+ item.setText(original_value)
+ return
+
+ try:
+ # Convert the string value to the appropriate type
+ if value_type == GGUFValueType.UINT8:
+ converted_value = np.uint8(int(new_value))
+ elif value_type == GGUFValueType.INT8:
+ converted_value = np.int8(int(new_value))
+ elif value_type == GGUFValueType.UINT16:
+ converted_value = np.uint16(int(new_value))
+ elif value_type == GGUFValueType.INT16:
+ converted_value = np.int16(int(new_value))
+ elif value_type == GGUFValueType.UINT32:
+ converted_value = np.uint32(int(new_value))
+ elif value_type == GGUFValueType.INT32:
+ converted_value = np.int32(int(new_value))
+ elif value_type == GGUFValueType.FLOAT32:
+ converted_value = np.float32(float(new_value))
+ elif value_type == GGUFValueType.BOOL:
+ converted_value = new_value.lower() in ('true', 'yes', '1')
+ elif value_type == GGUFValueType.STRING:
+ converted_value = new_value
+ else:
+ # Unsupported type for editing
+ return
+
+ # Store the change
+ self.metadata_changes[key] = (value_type, converted_value)
+ self.modified = True
+
+ self.statusBar().showMessage(f"Changed {key} to {new_value}")
+ except ValueError:
+ QMessageBox.warning(self, "Invalid Value", f"The value '{new_value}' is not valid for type {value_type.name}")
+
+ # Revert to original value
+ original_value = self.format_field_value(field)
+ item.setText(original_value)
+
+ def remove_metadata(self):
+ button = self.sender()
+ key = button.property("key")
+ row = button.property("row")
+
+ reply = QMessageBox.question(
+ self, "Confirm Removal",
+ f"Are you sure you want to remove the metadata key '{key}'?",
+ QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No
+ )
+
+ if reply == QMessageBox.StandardButton.Yes:
+ self.metadata_table.removeRow(row)
+ self.metadata_to_remove.add(key)
+
+ # If we previously had changes for this key, remove them
+ if key in self.metadata_changes:
+ del self.metadata_changes[key]
+
+ self.modified = True
+ self.statusBar().showMessage(f"Marked {key} for removal")
+
+ def edit_metadata_enum(self):
+ """Edit an enum metadata field."""
+ button = self.sender()
+ key = button.property("key")
+ row = button.property("row")
+
+ field = None
+ if self.reader:
+ field = self.reader.get_field(key)
+ if not field or not field.types:
+ return
+
+ enum_type = self.get_enum_for_key(key)
+ if enum_type is None:
+ return
+
+ # Get current value
+ current_value = field.contents()
+
+ # Create a dialog with enum options
+ dialog = QDialog(self)
+ dialog.setWindowTitle(f"Select {enum_type.__name__} Value")
+ layout = QVBoxLayout(dialog)
+
+ combo = QComboBox()
+ for enum_val in enum_type:
+ combo.addItem(f"{enum_val.name} ({enum_val.value})", enum_val.value)
+
+ # Set current value
+ try:
+ if isinstance(current_value, (int, str)):
+ enum_val = enum_type(current_value)
+ combo.setCurrentText(f"{enum_val.name} ({current_value})")
+ except (ValueError, KeyError):
+ pass
+
+ layout.addWidget(combo)
+
+ buttons = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel)
+ buttons.accepted.connect(dialog.accept)
+ buttons.rejected.connect(dialog.reject)
+ layout.addWidget(buttons)
+
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ # Get the selected value
+ new_value = combo.currentData()
+ enum_val = enum_type(new_value)
+
+ # Store the change
+ self.metadata_changes[key] = (field.types[0], new_value)
+ self.modified = True
+
+ # Update display
+ display_text = f"{enum_val.name} ({new_value})"
+ target_item = self.metadata_table.item(row, 2)
+ if target_item:
+ target_item.setText(display_text)
+
+ self.statusBar().showMessage(f"Changed {key} to {display_text}")
+
+ def edit_array_metadata(self):
+ button = self.sender()
+ key = button.property("key")
+ row = button.property("row")
+
+ # Check if this is one of the linked tokenizer keys
+ if key in TOKENIZER_LINKED_KEYS:
+ self.edit_tokenizer_metadata(key)
+ return
+
+ field = None
+ if self.reader:
+ field = self.reader.get_field(key)
+ if not field or not field.types or field.types[0] != GGUFValueType.ARRAY:
+ return
+
+ # Get array element type
+ element_type = field.types[1]
+
+ # Extract array values
+ array_values = self.extract_array_values(field)
+
+ # Open array editor dialog
+ dialog = ArrayEditorDialog(array_values, element_type, key, self)
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ new_values = dialog.get_array_values()
+
+ # Store the change
+ self.metadata_changes[key] = (GGUFValueType.ARRAY, (element_type, new_values))
+ self.modified = True
+
+ # Update display
+ enum_type = self.get_enum_for_key(key)
+ if enum_type is not None and element_type == GGUFValueType.INT32:
+ value_str = f"[ {', '.join(self.format_enum_value(v, enum_type) for v in new_values[:5])}{', ...' if len(new_values) > 5 else ''} ]"
+ else:
+ value_str = f"[ {', '.join(str(v) for v in new_values[:5])}{', ...' if len(new_values) > 5 else ''} ]"
+ target_item = self.metadata_table.item(row, 2)
+ if target_item:
+ target_item.setText(value_str)
+
+ self.statusBar().showMessage(f"Updated array values for {key}")
+
+ def edit_tokenizer_metadata(self, trigger_key):
+ """Edit the linked tokenizer metadata arrays together."""
+ if not self.reader:
+ return
+
+ # Get all three fields
+ tokens_field = self.reader.get_field(gguf.Keys.Tokenizer.LIST)
+ token_types_field = self.reader.get_field(gguf.Keys.Tokenizer.TOKEN_TYPE)
+ scores_field = self.reader.get_field(gguf.Keys.Tokenizer.SCORES)
+
+ # Extract values from each field
+ tokens = self.extract_array_values(tokens_field) if tokens_field else []
+ token_types = self.extract_array_values(token_types_field) if token_types_field else []
+ scores = self.extract_array_values(scores_field) if scores_field else []
+
+ # Apply any pending changes
+ if gguf.Keys.Tokenizer.LIST in self.metadata_changes:
+ _, (_, tokens) = self.metadata_changes[gguf.Keys.Tokenizer.LIST]
+ if gguf.Keys.Tokenizer.TOKEN_TYPE in self.metadata_changes:
+ _, (_, token_types) = self.metadata_changes[gguf.Keys.Tokenizer.TOKEN_TYPE]
+ if gguf.Keys.Tokenizer.SCORES in self.metadata_changes:
+ _, (_, scores) = self.metadata_changes[gguf.Keys.Tokenizer.SCORES]
+
+ # Open the tokenizer editor dialog
+ dialog = TokenizerEditorDialog(tokens, token_types, scores, self)
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ new_tokens, new_token_types, new_scores = dialog.get_data()
+
+ # Store changes for all three arrays
+ if tokens_field:
+ self.metadata_changes[gguf.Keys.Tokenizer.LIST] = (
+ GGUFValueType.ARRAY,
+ (tokens_field.types[1], new_tokens)
+ )
+
+ if token_types_field:
+ self.metadata_changes[gguf.Keys.Tokenizer.TOKEN_TYPE] = (
+ GGUFValueType.ARRAY,
+ (token_types_field.types[1], new_token_types)
+ )
+
+ if scores_field:
+ self.metadata_changes[gguf.Keys.Tokenizer.SCORES] = (
+ GGUFValueType.ARRAY,
+ (scores_field.types[1], new_scores)
+ )
+
+ self.modified = True
+
+ # Update display for all three fields
+ self.update_tokenizer_display(gguf.Keys.Tokenizer.LIST, new_tokens)
+ self.update_tokenizer_display(gguf.Keys.Tokenizer.TOKEN_TYPE, new_token_types)
+ self.update_tokenizer_display(gguf.Keys.Tokenizer.SCORES, new_scores)
+
+ self.statusBar().showMessage("Updated tokenizer data")
+
+ def update_tokenizer_display(self, key, values):
+ """Update the display of a tokenizer field in the metadata table."""
+ for row in range(self.metadata_table.rowCount()):
+ key_item = self.metadata_table.item(row, 0)
+ if key_item and key_item.text() == key:
+ value_str = f"[ {', '.join(str(v) for v in values[:5])}{', ...' if len(values) > 5 else ''} ]"
+ value_item = self.metadata_table.item(row, 2)
+ if value_item:
+ value_item.setText(value_str)
+ break
+
+ def add_metadata(self):
+ dialog = AddMetadataDialog(self)
+ if dialog.exec() == QDialog.DialogCode.Accepted:
+ key, value_type, value = dialog.get_data()
+
+ if not key:
+ QMessageBox.warning(self, "Invalid Key", "Key cannot be empty")
+ return
+
+ # Check if key already exists
+ for row in range(self.metadata_table.rowCount()):
+ orig_item = self.metadata_table.item(row, 0)
+ if orig_item and orig_item.text() == key:
+ QMessageBox.warning(self, "Duplicate Key", f"Key '{key}' already exists")
+ return
+
+ # Add to table
+ row = self.metadata_table.rowCount()
+ self.metadata_table.insertRow(row)
+
+ # Key
+ key_item = QTableWidgetItem(key)
+ key_item.setFlags(key_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.metadata_table.setItem(row, 0, key_item)
+
+ # Type
+ type_item = QTableWidgetItem(value_type.name)
+ type_item.setFlags(type_item.flags() & ~Qt.ItemFlag.ItemIsEditable)
+ self.metadata_table.setItem(row, 1, type_item)
+
+ # Value
+ value_item = QTableWidgetItem(str(value))
+ value_item.setFlags(value_item.flags() | Qt.ItemFlag.ItemIsEditable)
+ self.metadata_table.setItem(row, 2, value_item)
+
+ # Actions
+ actions_widget = QWidget()
+ actions_layout = QHBoxLayout(actions_widget)
+ actions_layout.setContentsMargins(2, 2, 2, 2)
+
+ remove_button = QPushButton("Remove")
+ remove_button.setProperty("row", row)
+ remove_button.setProperty("key", key)
+ remove_button.clicked.connect(self.remove_metadata)
+ actions_layout.addWidget(remove_button)
+
+ self.metadata_table.setCellWidget(row, 3, actions_widget)
+
+ # Store the change
+ self.metadata_changes[key] = (value_type, value)
+ self.modified = True
+
+ self.statusBar().showMessage(f"Added new metadata key {key}")
+
+ def save_file(self):
+ if not self.reader:
+ QMessageBox.warning(self, "No File Open", "Please open a GGUF file first")
+ return
+
+ if not self.modified and not self.metadata_changes and not self.metadata_to_remove:
+ QMessageBox.information(self, "No Changes", "No changes to save")
+ return
+
+ file_path, _ = QFileDialog.getSaveFileName(
+ self, "Save GGUF File As", "", "GGUF Files (*.gguf);;All Files (*)"
+ )
+
+ if not file_path:
+ return
+
+ try:
+ self.statusBar().showMessage(f"Saving to {file_path}...")
+ QApplication.processEvents()
+
+ # Get architecture and endianness from the original file
+ arch = 'unknown'
+ field = self.reader.get_field(gguf.Keys.General.ARCHITECTURE)
+ if field:
+ arch = field.contents()
+
+ # Create writer
+ writer = GGUFWriter(file_path, arch=arch, endianess=self.reader.endianess)
+
+ # Get alignment if present
+ alignment = None
+ field = self.reader.get_field(gguf.Keys.General.ALIGNMENT)
+ if field:
+ alignment = field.contents()
+ if alignment is not None:
+ writer.data_alignment = alignment
+
+ # Copy metadata with changes
+ for field in self.reader.fields.values():
+ # Skip virtual fields and fields written by GGUFWriter
+ if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'):
+ continue
+
+ # Skip fields marked for removal
+ if field.name in self.metadata_to_remove:
+ continue
+
+ # Apply changes if any
+ if field.name in self.metadata_changes:
+ value_type, value = self.metadata_changes[field.name]
+ if value_type == GGUFValueType.ARRAY:
+ # Handle array values
+ element_type, array_values = value
+ writer.add_array(field.name, array_values)
+ else:
+ writer.add_key_value(field.name, value, value_type)
+ else:
+ # Copy original value
+ value = field.contents()
+ if value is not None and field.types:
+ writer.add_key_value(field.name, value, field.types[0])
+
+ # Add new metadata
+ for key, (value_type, value) in self.metadata_changes.items():
+ # Skip if the key already existed (we handled it above)
+ if self.reader.get_field(key) is not None:
+ continue
+
+ writer.add_key_value(key, value, value_type)
+
+ # Add tensors (including data)
+ for tensor in self.reader.tensors:
+ writer.add_tensor(tensor.name, tensor.data, raw_shape=tensor.data.shape, raw_dtype=tensor.tensor_type)
+
+ # Write header and metadata
+ writer.open_output_file(Path(file_path))
+ writer.write_header_to_file()
+ writer.write_kv_data_to_file()
+
+ # Write tensor data using the optimized method
+ writer.write_tensors_to_file(progress=False)
+
+ writer.close()
+
+ self.statusBar().showMessage(f"Saved to {file_path}")
+
+ # Ask if user wants to open the new file
+ reply = QMessageBox.question(
+ self, "Open Saved File",
+ "Would you like to open the newly saved file?",
+ QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.Yes
+ )
+
+ if reply == QMessageBox.StandardButton.Yes:
+ self.reader = GGUFReader(file_path, 'r')
+ self.current_file = file_path
+ self.file_path_edit.setText(file_path)
+
+ self.load_metadata()
+ self.load_tensors()
+
+ self.metadata_changes = {}
+ self.metadata_to_remove = set()
+ self.modified = False
+
+ except Exception as e:
+ QMessageBox.critical(self, "Error", f"Failed to save file: {str(e)}")
+ self.statusBar().showMessage("Error saving file")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="GUI GGUF Editor")
+ parser.add_argument("model_path", nargs="?", help="path to GGUF model file to load at startup")
+ parser.add_argument("--verbose", action="store_true", help="increase output verbosity")
+
+ args = parser.parse_args()
+
+ logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+
+ app = QApplication(sys.argv)
+ window = GGUFEditorWindow()
+ window.show()
+
+ # Load model if specified
+ if args.model_path:
+ if os.path.isfile(args.model_path) and args.model_path.endswith('.gguf'):
+ window.load_file(args.model_path)
+ else:
+ logger.error(f"Invalid model path: {args.model_path}")
+ QMessageBox.warning(
+ window,
+ "Invalid Model Path",
+ f"The specified file does not exist or is not a GGUF file: {args.model_path}")
+
+ sys.exit(app.exec())
+
+
+if __name__ == '__main__':
+ main()
overview / pkg / ggml / sources / llama.cpp / commitdiff