From: Georgi Gerganov Date: Tue, 11 Aug 2026 06:07:13 +0000 (+0300) Subject: tests : clean-up server test, use `tests.sh` in ci (#26886) X-Git-Tag: upstream/0.0.10438~75 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=b3df57286c336255fd02c2162b0b5fe3adc2965e;p=pkg%2Fggml%2Fsources%2Fllama.cpp tests : clean-up server test, use `tests.sh` in ci (#26886) * tests : remove fetch_server_test_models.py * ci : use tests.sh wrapper of pytest --- diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml index 0eeefdf88..5d696282c 100644 --- a/.github/workflows/server-sanitize.yml +++ b/.github/workflows/server-sanitize.yml @@ -110,7 +110,7 @@ jobs: source .venv/bin/activate cd tools/server/tests export ${{ matrix.extra_args }} - pytest -v -x -m "not slow" + ./tests.sh - name: Slow tests id: server_integration_tests_slow @@ -119,4 +119,4 @@ jobs: source .venv/bin/activate cd tools/server/tests export ${{ matrix.extra_args }} - SLOW_TESTS=1 pytest -v -x + SLOW_TESTS=1 ./tests.sh diff --git a/.github/workflows/server-self-hosted.yml b/.github/workflows/server-self-hosted.yml index 249f389ff..675ddbaaa 100644 --- a/.github/workflows/server-self-hosted.yml +++ b/.github/workflows/server-self-hosted.yml @@ -72,7 +72,7 @@ jobs: run: | cd tools/server/tests source venv/bin/activate - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx1, backend-sampling) id: server_integration_tests_backend_sampling @@ -81,7 +81,7 @@ jobs: cd tools/server/tests source venv/bin/activate export LLAMA_ARG_BACKEND_SAMPLING=1 - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx2) id: server_integration_tests_gpu2 @@ -90,7 +90,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_METAL_DEVICES=2 - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx2, backend-sampling) id: server_integration_tests_gpu2_backend_sampling @@ -99,7 +99,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_METAL_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 - pytest -v -x -m "not slow" + ./tests.sh server-cuda: runs-on: [self-hosted, llama-server, Linux, NVIDIA] @@ -132,7 +132,7 @@ jobs: run: | cd tools/server/tests source venv/bin/activate - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx1, backend-sampling) id: server_integration_tests_backend_sampling @@ -141,7 +141,7 @@ jobs: cd tools/server/tests source venv/bin/activate export LLAMA_ARG_BACKEND_SAMPLING=1 - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx2) id: server_integration_tests_gpu2 @@ -150,7 +150,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_CUDA_DEVICES=2 - pytest -v -x -m "not slow" + ./tests.sh - name: Tests (GPUx2, backend-sampling) id: server_integration_tests_gpu2_backend_sampling @@ -159,7 +159,7 @@ jobs: cd tools/server/tests source venv/bin/activate export GGML_CUDA_DEVICES=2 LLAMA_ARG_BACKEND_SAMPLING=1 - pytest -v -x -m "not slow" + ./tests.sh server-kleidiai: runs-on: ah-ubuntu_22_04-c8g_8x @@ -219,4 +219,4 @@ jobs: run: | cd tools/server/tests source venv/bin/activate - pytest -v -x -m "not slow" + ./tests.sh diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 5a02cc15a..d5abf1d23 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -104,21 +104,21 @@ jobs: id: server_integration_tests run: | cd tools/server/tests - pytest -v -x -m "not slow" + ./tests.sh - name: Slow tests id: server_integration_tests_slow if: ${{ github.event.schedule || github.event.inputs.slow_tests == 'true' }} run: | cd tools/server/tests - SLOW_TESTS=1 pytest -v -x + SLOW_TESTS=1 ./tests.sh - name: Tests (Backend sampling) id: server_integration_tests_backend_sampling run: | cd tools/server/tests export LLAMA_ARG_BACKEND_SAMPLING=1 - pytest -v -x -m "not slow" + ./tests.sh - name: Slow tests (Backend sampling) id: server_integration_tests_slow_backend_sampling @@ -126,7 +126,7 @@ jobs: run: | cd tools/server/tests export LLAMA_ARG_BACKEND_SAMPLING=1 - SLOW_TESTS=1 pytest -v -x + SLOW_TESTS=1 ./tests.sh windows: runs-on: windows-2025 @@ -170,7 +170,7 @@ jobs: run: | cd tools/server/tests $env:PYTHONIOENCODING = ":replace" - pytest -v -x -m "not slow" + ./tests.sh - name: Slow tests id: server_integration_tests_slow @@ -178,4 +178,4 @@ jobs: run: | cd tools/server/tests $env:SLOW_TESTS = "1" - pytest -v -x + ./tests.sh diff --git a/scripts/fetch_server_test_models.py b/scripts/fetch_server_test_models.py deleted file mode 100755 index f43d1f63c..000000000 --- a/scripts/fetch_server_test_models.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -''' - This script fetches all the models used in the server tests. - - This is useful for slow tests that use larger models, to avoid them timing out on the model downloads. - - It is meant to be run from the root of the repository. - - Example: - python scripts/fetch_server_test_models.py - ( cd tools/server/tests && ./tests.sh -v -x -m slow ) -''' -import ast -import glob -import logging -import os -from typing import Generator -from pydantic import BaseModel -from typing import Optional -import subprocess - - -class HuggingFaceModel(BaseModel): - hf_repo: str - hf_file: Optional[str] = None - - class Config: - frozen = True - - -def collect_hf_model_test_parameters(test_file) -> Generator[HuggingFaceModel, None, None]: - try: - with open(test_file) as f: - tree = ast.parse(f.read()) - except Exception as e: - logging.error(f'collect_hf_model_test_parameters failed on {test_file}: {e}') - return - - for node in ast.walk(tree): - if isinstance(node, ast.FunctionDef): - for dec in node.decorator_list: - if isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute) and dec.func.attr == 'parametrize': - param_names = ast.literal_eval(dec.args[0]).split(",") - if "hf_repo" not in param_names: - continue - - raw_param_values = dec.args[1] - if not isinstance(raw_param_values, ast.List): - logging.warning(f'Skipping non-list parametrize entry at {test_file}:{node.lineno}') - continue - - hf_repo_idx = param_names.index("hf_repo") - hf_file_idx = param_names.index("hf_file") if "hf_file" in param_names else None - - for t in raw_param_values.elts: - if not isinstance(t, ast.Tuple): - logging.warning(f'Skipping non-tuple parametrize entry at {test_file}:{node.lineno}') - continue - yield HuggingFaceModel( - hf_repo=ast.literal_eval(t.elts[hf_repo_idx]), - hf_file=ast.literal_eval(t.elts[hf_file_idx]) if hf_file_idx is not None else None) - - -if __name__ == '__main__': - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - - models = sorted(list(set([ - model - for test_file in glob.glob('tools/server/tests/unit/test_*.py') - for model in collect_hf_model_test_parameters(test_file) - ])), key=lambda m: (m.hf_repo, m.hf_file)) - - logging.info(f'Found {len(models)} models in parameterized tests:') - for m in models: - logging.info(f' - {m.hf_repo} / {m.hf_file}') - - cli_path = os.environ.get( - 'LLAMA_CLI_BIN_PATH', - os.path.join( - os.path.dirname(__file__), - '../build/bin/Release/llama-cli.exe' if os.name == 'nt' else '../build/bin/llama-cli')) - - for m in models: - if '<' in m.hf_repo or (m.hf_file is not None and '<' in m.hf_file): - continue - if m.hf_file is not None and '-of-' in m.hf_file: - logging.warning(f'Skipping model at {m.hf_repo} / {m.hf_file} because it is a split file') - continue - logging.info(f'Using llama-cli to ensure model {m.hf_repo}/{m.hf_file} was fetched') - cmd = [ - cli_path, - '-hfr', m.hf_repo, - *([] if m.hf_file is None else ['-hff', m.hf_file]), - '-n', '1', - '-p', 'Hey', - '--no-warmup', - '--log-disable', - '-st'] - if m.hf_file != 'tinyllamas/stories260K.gguf' and 'Mistral-Nemo' not in m.hf_repo: - cmd += ('-fa', 'on') - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - logging.error(f'Failed to fetch model at {m.hf_repo} / {m.hf_file} with command:\n {" ".join(cmd)}') - exit(1) diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.sh index 8d6681193..433dc9982 100755 --- a/tools/server/tests/tests.sh +++ b/tools/server/tests/tests.sh @@ -6,11 +6,6 @@ cd $SCRIPT_DIR set -eu -if [[ "${SLOW_TESTS:-0}" == 1 ]]; then - # Slow tests for tool calls need quite a few models ahead of time to avoid timing out. - python $SCRIPT_DIR/../../../scripts/fetch_server_test_models.py -fi - if [ $# -lt 1 ] then if [[ "${SLOW_TESTS:-0}" == 1 ]]; then