uses: actions/setup-python@v6
with:
python-version: "3.11"
- pip-install: -r requirements/requirements-all.txt ty==0.0.24
+ pip-install: -r requirements/requirements-all.txt ty==0.0.26
# - name: Type-check with Pyright
# uses: jakebailey/pyright-action@v2
# with:
from gguf.vocab import MistralTokenizerType, MistralVocab
try:
- from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found]
- from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found]
- from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found]
- from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.base import TokenizerVersion # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.multimodal import DATASET_MEAN as _MISTRAL_COMMON_DATASET_MEAN, DATASET_STD as _MISTRAL_COMMON_DATASET_STD # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
SentencePieceTokenizer,
)
# Add utils directory to path for direct script execution
sys.path.insert(0, str(Path(__file__).parent.parent / "utils"))
-from common import get_model_name_from_env_path, compare_tokens, exit_with_warning # type: ignore[import-not-found]
+from common import get_model_name_from_env_path, compare_tokens, exit_with_warning # type: ignore[import-not-found, ty:unresolved-import]
def quick_logits_check(pytorch_file, llamacpp_file):
"""Lightweight sanity check before NMSE"""
import os
import argparse
from pathlib import Path
-from common import get_model_name_from_env_path # type: ignore[import-not-found]
+from common import get_model_name_from_env_path # type: ignore[import-not-found, ty:unresolved-import]
def calculate_nmse(reference, test):
mse = np.mean((test - reference) ** 2)
import argparse
import sys
-from common import compare_tokens # type: ignore[import-not-found]
+from common import compare_tokens # type: ignore[import-not-found, ty:unresolved-import]
def parse_arguments():
from pathlib import Path
from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, AutoModel
-from common import compare_tokens, exit_with_warning # type: ignore[import-not-found]
+from common import compare_tokens, exit_with_warning # type: ignore[import-not-found, ty:unresolved-import]
unreleased_model_name = os.getenv('UNRELEASED_MODEL_NAME')
SentencePieceProcessor: Any = None
try:
- from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found]
- from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found]
- from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
+ from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
_filter_valid_tokenizer_files,
)
- from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
SentencePieceTokenizer,
)
except ImportError:
_mistral_common_installed = True
try:
- from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found]
+ from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
get_one_valid_tokenizer_file,
)
except ImportError:
for codepoint, norm in table_nfd:
start = ranges_nfd[-1][0]
if ranges_nfd[-1] != (start, codepoint - 1, norm):
- ranges_nfd.append(None) # type: ignore[arg-type] # dummy, will be replaced below
+ ranges_nfd.append((0, 0, 0)) # dummy, will be replaced below
start = codepoint
ranges_nfd[-1] = (start, codepoint, norm)
self.server_port = int(os.environ["PORT"])
self.external_server = "DEBUG_EXTERNAL" in os.environ
- def start(self, timeout_seconds: int | None = DEFAULT_HTTP_TIMEOUT) -> None:
+ def start(self, timeout_seconds: int = DEFAULT_HTTP_TIMEOUT) -> None:
if self.external_server:
print(f"[external_server]: Assuming external server running on {self.server_host}:{self.server_port}")
return