Signed-off-by: Jiri Podivin <redacted>
Co-authored-by: Jiri Podivin <redacted>
import argparse
import json
import os
-import struct
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Any
-import itertools
import numpy as np
import torch
from sentencepiece import SentencePieceProcessor # type: ignore[import]
from __future__ import annotations
import argparse
-import math
import struct
import sys
from enum import IntEnum
import argparse
import gguf
-import os
import struct
-import sys
import numpy as np
from pathlib import Path
# tests with BPE tokenizer
-import os
-import sys
import argparse
from transformers import AutoTokenizer
# tests with SPM tokenizer
-import os
-import sys
import argparse
from sentencepiece import SentencePieceProcessor