--- /dev/null
+#!/usr/bin/env python3
+import sys
+import json
+import argparse
+import jinja2.ext as jinja2_ext
+from PySide6.QtWidgets import (
+ QApplication,
+ QMainWindow,
+ QWidget,
+ QVBoxLayout,
+ QHBoxLayout,
+ QLabel,
+ QPlainTextEdit,
+ QTextEdit,
+ QPushButton,
+ QFileDialog,
+)
+from PySide6.QtGui import QColor, QColorConstants, QTextCursor, QTextFormat
+from PySide6.QtCore import Qt, QRect, QSize
+from jinja2 import TemplateSyntaxError
+from jinja2.sandbox import ImmutableSandboxedEnvironment
+from datetime import datetime
+
+
+def format_template_content(template_content):
+ """Format the Jinja template content using Jinja2's lexer."""
+ if not template_content.strip():
+ return template_content
+
+ env = ImmutableSandboxedEnvironment()
+ tc_rstrip = template_content.rstrip()
+ tokens = list(env.lex(tc_rstrip))
+ result = ""
+ indent_level = 0
+ i = 0
+
+ while i < len(tokens):
+ token = tokens[i]
+ _, token_type, token_value = token
+
+ if token_type == "block_begin":
+ block_start = i
+ # Collect all tokens for this block construct
+ construct_content = token_value
+ end_token_type = token_type.replace("_begin", "_end")
+ j = i + 1
+ while j < len(tokens) and tokens[j][1] != end_token_type:
+ construct_content += tokens[j][2]
+ j += 1
+
+ if j < len(tokens): # Found the end token
+ construct_content += tokens[j][2]
+ i = j # Skip to the end token
+
+ # Check for control structure keywords for indentation
+ stripped_content = construct_content.strip()
+ instr = block_start + 1
+ while tokens[instr][1] == "whitespace":
+ instr = instr + 1
+
+ instruction_token = tokens[instr][2]
+ start_control_tokens = ["if", "for", "macro", "call", "block"]
+ end_control_tokens = ["end" + t for t in start_control_tokens]
+ is_control_start = any(
+ instruction_token.startswith(kw) for kw in start_control_tokens
+ )
+ is_control_end = any(
+ instruction_token.startswith(kw) for kw in end_control_tokens
+ )
+
+ # Adjust indentation for control structures
+ # For control end blocks, decrease indent BEFORE adding the content
+ if is_control_end:
+ indent_level = max(0, indent_level - 1)
+
+ # Remove all previous whitespace before this block
+ result = result.rstrip()
+
+ # Add proper indent, but only if this is not the first token
+ added_newline = False
+ if result: # Only add newline and indent if there's already content
+ result += (
+ "\n" + " " * indent_level
+ ) # Use 2 spaces per indent level
+ added_newline = True
+ else: # For the first token, don't add any indent
+ result += ""
+
+ # Add the block content
+ result += stripped_content
+
+ # Add '-' after '%' if it wasn't there and we added a newline or indent
+ if (
+ added_newline
+ and stripped_content.startswith("{%")
+ and not stripped_content.startswith("{%-")
+ ):
+ # Add '-' at the beginning
+ result = (
+ result[: result.rfind("{%")]
+ + "{%-"
+ + result[result.rfind("{%") + 2 :]
+ )
+ if stripped_content.endswith("%}") and not stripped_content.endswith(
+ "-%}"
+ ):
+ # Only add '-' if this is not the last token or if there's content after
+ if i + 1 < len(tokens) and tokens[i + 1][1] != "eof":
+ result = result[:-2] + "-%}"
+
+ # For control start blocks, increase indent AFTER adding the content
+ if is_control_start:
+ indent_level += 1
+ else:
+ # Malformed template, just add the token
+ result += token_value
+ elif token_type == "variable_begin":
+ # Collect all tokens for this variable construct
+ construct_content = token_value
+ end_token_type = token_type.replace("_begin", "_end")
+ j = i + 1
+ while j < len(tokens) and tokens[j][1] != end_token_type:
+ construct_content += tokens[j][2]
+ j += 1
+
+ if j < len(tokens): # Found the end token
+ construct_content += tokens[j][2]
+ i = j # Skip to the end token
+
+ # For variable constructs, leave them alone
+ # Do not add indent or whitespace before or after them
+ result += construct_content
+ else:
+ # Malformed template, just add the token
+ result += token_value
+ elif token_type == "data":
+ # Handle data (text between Jinja constructs)
+ # For data content, preserve it as is
+ result += token_value
+ else:
+ # Handle any other tokens
+ result += token_value
+
+ i += 1
+
+ # Clean up trailing newlines and spaces
+ result = result.rstrip()
+
+ # Copy the newline / space count from the original
+ if (trailing_length := len(template_content) - len(tc_rstrip)):
+ result += template_content[-trailing_length:]
+
+ return result
+
+
+# ------------------------
+# Line Number Widget
+# ------------------------
+class LineNumberArea(QWidget):
+ def __init__(self, editor):
+ super().__init__(editor)
+ self.code_editor = editor
+
+ def sizeHint(self):
+ return QSize(self.code_editor.line_number_area_width(), 0)
+
+ def paintEvent(self, event):
+ self.code_editor.line_number_area_paint_event(event)
+
+
+class CodeEditor(QPlainTextEdit):
+ def __init__(self):
+ super().__init__()
+ self.line_number_area = LineNumberArea(self)
+
+ self.blockCountChanged.connect(self.update_line_number_area_width)
+ self.updateRequest.connect(self.update_line_number_area)
+ self.cursorPositionChanged.connect(self.highlight_current_line)
+
+ self.update_line_number_area_width(0)
+ self.highlight_current_line()
+
+ def line_number_area_width(self):
+ digits = len(str(self.blockCount()))
+ space = 3 + self.fontMetrics().horizontalAdvance("9") * digits
+ return space
+
+ def update_line_number_area_width(self, _):
+ self.setViewportMargins(self.line_number_area_width(), 0, 0, 0)
+
+ def update_line_number_area(self, rect, dy):
+ if dy:
+ self.line_number_area.scroll(0, dy)
+ else:
+ self.line_number_area.update(
+ 0, rect.y(), self.line_number_area.width(), rect.height()
+ )
+
+ if rect.contains(self.viewport().rect()):
+ self.update_line_number_area_width(0)
+
+ def resizeEvent(self, event):
+ super().resizeEvent(event)
+ cr = self.contentsRect()
+ self.line_number_area.setGeometry(
+ QRect(cr.left(), cr.top(), self.line_number_area_width(), cr.height())
+ )
+
+ def line_number_area_paint_event(self, event):
+ from PySide6.QtGui import QPainter
+
+ painter = QPainter(self.line_number_area)
+ painter.fillRect(event.rect(), QColorConstants.LightGray)
+
+ block = self.firstVisibleBlock()
+ block_number = block.blockNumber()
+ top = int(
+ self.blockBoundingGeometry(block).translated(self.contentOffset()).top()
+ )
+ bottom = top + int(self.blockBoundingRect(block).height())
+
+ while block.isValid() and top <= event.rect().bottom():
+ if block.isVisible() and bottom >= event.rect().top():
+ number = str(block_number + 1)
+ painter.setPen(QColorConstants.Black)
+ painter.drawText(
+ 0,
+ top,
+ self.line_number_area.width() - 2,
+ self.fontMetrics().height(),
+ Qt.AlignmentFlag.AlignRight,
+ number,
+ )
+ block = block.next()
+ top = bottom
+ bottom = top + int(self.blockBoundingRect(block).height())
+ block_number += 1
+
+ def highlight_current_line(self):
+ extra_selections = []
+ if not self.isReadOnly():
+ selection = QTextEdit.ExtraSelection()
+ line_color = QColorConstants.Yellow.lighter(160)
+ selection.format.setBackground(line_color) # pyright: ignore[reportAttributeAccessIssue]
+ selection.format.setProperty(QTextFormat.Property.FullWidthSelection, True) # pyright: ignore[reportAttributeAccessIssue]
+ selection.cursor = self.textCursor() # pyright: ignore[reportAttributeAccessIssue]
+ selection.cursor.clearSelection() # pyright: ignore[reportAttributeAccessIssue]
+ extra_selections.append(selection)
+ self.setExtraSelections(extra_selections)
+
+ def highlight_position(self, lineno: int, col: int, color: QColor):
+ block = self.document().findBlockByLineNumber(lineno - 1)
+ if block.isValid():
+ cursor = QTextCursor(block)
+ text = block.text()
+ start = block.position() + max(0, col - 1)
+ cursor.setPosition(start)
+ if col <= len(text):
+ cursor.movePosition(
+ QTextCursor.MoveOperation.NextCharacter,
+ QTextCursor.MoveMode.KeepAnchor,
+ )
+
+ extra = QTextEdit.ExtraSelection()
+ extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue]
+ extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue]
+
+ self.setExtraSelections(self.extraSelections() + [extra])
+
+ def highlight_line(self, lineno: int, color: QColor):
+ block = self.document().findBlockByLineNumber(lineno - 1)
+ if block.isValid():
+ cursor = QTextCursor(block)
+ cursor.select(QTextCursor.SelectionType.LineUnderCursor)
+
+ extra = QTextEdit.ExtraSelection()
+ extra.format.setBackground(color.lighter(160)) # pyright: ignore[reportAttributeAccessIssue]
+ extra.cursor = cursor # pyright: ignore[reportAttributeAccessIssue]
+
+ self.setExtraSelections(self.extraSelections() + [extra])
+
+ def clear_highlighting(self):
+ self.highlight_current_line()
+
+
+# ------------------------
+# Main App
+# ------------------------
+class JinjaTester(QMainWindow):
+ def __init__(self):
+ super().__init__()
+ self.setWindowTitle("Jinja Template Tester")
+ self.resize(1200, 800)
+
+ central = QWidget()
+ main_layout = QVBoxLayout(central)
+
+ # -------- Top input area --------
+ input_layout = QHBoxLayout()
+
+ # Template editor with label
+ template_layout = QVBoxLayout()
+ template_label = QLabel("Jinja2 Template")
+ template_layout.addWidget(template_label)
+ self.template_edit = CodeEditor()
+ template_layout.addWidget(self.template_edit)
+ input_layout.addLayout(template_layout)
+
+ # JSON editor with label
+ json_layout = QVBoxLayout()
+ json_label = QLabel("Context (JSON)")
+ json_layout.addWidget(json_label)
+ self.json_edit = CodeEditor()
+ self.json_edit.setPlainText("""
+{
+ "add_generation_prompt": true,
+ "bos_token": "",
+ "eos_token": "",
+ "messages": [
+ {
+ "role": "user",
+ "content": "What is the capital of Poland?"
+ }
+ ]
+}
+ """.strip())
+ json_layout.addWidget(self.json_edit)
+ input_layout.addLayout(json_layout)
+
+ main_layout.addLayout(input_layout)
+
+ # -------- Rendered output area --------
+ output_label = QLabel("Rendered Output")
+ main_layout.addWidget(output_label)
+ self.output_edit = QPlainTextEdit()
+ self.output_edit.setReadOnly(True)
+ main_layout.addWidget(self.output_edit)
+
+ # -------- Render button and status --------
+ btn_layout = QHBoxLayout()
+
+ # Load template button
+ self.load_btn = QPushButton("Load Template")
+ self.load_btn.clicked.connect(self.load_template)
+ btn_layout.addWidget(self.load_btn)
+
+ # Format template button
+ self.format_btn = QPushButton("Format")
+ self.format_btn.clicked.connect(self.format_template)
+ btn_layout.addWidget(self.format_btn)
+
+ self.render_btn = QPushButton("Render")
+ self.render_btn.clicked.connect(self.render_template)
+ btn_layout.addWidget(self.render_btn)
+ main_layout.addLayout(btn_layout)
+
+ # Status label below buttons
+ self.status_label = QLabel("Ready")
+ main_layout.addWidget(self.status_label)
+
+ self.setCentralWidget(central)
+
+ def render_template(self):
+ self.template_edit.clear_highlighting()
+ self.output_edit.clear()
+
+ template_str = self.template_edit.toPlainText()
+ json_str = self.json_edit.toPlainText()
+
+ # Parse JSON context
+ try:
+ context = json.loads(json_str) if json_str.strip() else {}
+ except Exception as e:
+ self.status_label.setText(f"❌ JSON Error: {e}")
+ return
+
+ def raise_exception(text: str) -> str:
+ raise RuntimeError(text)
+
+ env = ImmutableSandboxedEnvironment(
+ trim_blocks=True,
+ lstrip_blocks=True,
+ extensions=[jinja2_ext.loopcontrols],
+ )
+ env.filters["tojson"] = (
+ lambda x,
+ indent=None,
+ separators=None,
+ sort_keys=False,
+ ensure_ascii=False: json.dumps(
+ x,
+ indent=indent,
+ separators=separators,
+ sort_keys=sort_keys,
+ ensure_ascii=ensure_ascii,
+ )
+ )
+ env.globals["strftime_now"] = lambda format: datetime.now().strftime(format)
+ env.globals["raise_exception"] = raise_exception
+ try:
+ template = env.from_string(template_str)
+ output = template.render(context)
+ self.output_edit.setPlainText(output)
+ self.status_label.setText("✅ Render successful")
+ except TemplateSyntaxError as e:
+ self.status_label.setText(f"❌ Syntax Error (line {e.lineno}): {e.message}")
+ if e.lineno:
+ self.template_edit.highlight_line(e.lineno, QColor("red"))
+ except Exception as e:
+ # Catch all runtime errors
+ # Try to extract template line number
+ lineno = None
+ tb = e.__traceback__
+ while tb:
+ frame = tb.tb_frame
+ if frame.f_code.co_filename == "<template>":
+ lineno = tb.tb_lineno
+ break
+ tb = tb.tb_next
+
+ error_msg = f"Runtime Error: {type(e).__name__}: {e}"
+ if lineno:
+ error_msg = f"Runtime Error at line {lineno} in template: {type(e).__name__}: {e}"
+ self.template_edit.highlight_line(lineno, QColor("orange"))
+
+ self.output_edit.setPlainText(error_msg)
+ self.status_label.setText(f"❌ {error_msg}")
+
+ def load_template(self):
+ """Load a Jinja template from a file using a file dialog."""
+ file_path, _ = QFileDialog.getOpenFileName(
+ self,
+ "Load Jinja Template",
+ "",
+ "Template Files (*.jinja *.j2 *.html *.txt);;All Files (*)",
+ )
+
+ if file_path:
+ try:
+ with open(file_path, "r", encoding="utf-8") as file:
+ content = file.read()
+ self.template_edit.setPlainText(content)
+ self.status_label.setText(f"✅ Loaded template from {file_path}")
+ except Exception as e:
+ self.status_label.setText(f"❌ Error loading file: {str(e)}")
+
+ def format_template(self):
+ """Format the Jinja template using Jinja2's lexer for proper parsing."""
+ try:
+ template_content = self.template_edit.toPlainText()
+ if not template_content.strip():
+ self.status_label.setText("⚠️ Template is empty")
+ return
+
+ formatted_content = format_template_content(template_content)
+ self.template_edit.setPlainText(formatted_content)
+ self.status_label.setText("✅ Template formatted")
+ except Exception as e:
+ self.status_label.setText(f"❌ Error formatting template: {str(e)}")
+
+
+if __name__ == "__main__":
+ if len(sys.argv) > 1:
+ # CLI mode
+ parser = argparse.ArgumentParser(description="Jinja Template Tester")
+ parser.add_argument(
+ "--template", required=True, help="Path to Jinja template file"
+ )
+ parser.add_argument("--context", required=True, help="JSON string for context")
+ parser.add_argument(
+ "--action",
+ choices=["format", "render"],
+ default="render",
+ help="Action to perform",
+ )
+ args = parser.parse_args()
+
+ # Load template
+ with open(args.template, "r", encoding="utf-8") as f:
+ template_content = f.read()
+
+ # Load JSON
+ context = json.loads(args.context)
+ # Add missing variables
+ context.setdefault("bos_token", "")
+ context.setdefault("eos_token", "")
+ context.setdefault("add_generation_prompt", False)
+
+ env = ImmutableSandboxedEnvironment()
+
+ if args.action == "format":
+ formatted = format_template_content(template_content)
+ print(formatted) # noqa: NP100
+ elif args.action == "render":
+ template = env.from_string(template_content)
+ output = template.render(context)
+ print(output) # noqa: NP100
+
+ else:
+ # GUI mode
+ app = QApplication(sys.argv)
+ window = JinjaTester()
+ window.show()
+ sys.exit(app.exec())
overview / pkg / ggml / sources / llama.cpp / commitdiff