ci : move sanitizer jobs to self-hosted runners (#23713)

diff --git a/.github/workflows/build-sanitize.yml b/.github/workflows/build-sanitize.yml
index c7b73d1dd0df4df174ea3f18c6927d3ac4c8d639..30430f71b1fe67d520d63f2eb7d90c04e903ff94 100644 (file)
--- a/.github/workflows/build-sanitize.yml
+++ b/.github/workflows/build-sanitize.yml
@@ -27,61 +27,70 @@ env:
   LLAMA_LOG_TIMESTAMPS: 1
 
 jobs:
-  ubuntu-latest-sanitizer:
-    runs-on: ubuntu-latest
+  ctest:
+    runs-on: [self-hosted, CPU, Linux]
 
     continue-on-error: true
 
     strategy:
       matrix:
         sanitizer: [ADDRESS, THREAD, UNDEFINED]
-        build_type: [Debug]
 
     steps:
       - name: Clone
         id: checkout
         uses: actions/checkout@v6
 
-      - name: ccache
-        uses: ggml-org/ccache-action@v1.2.21
-        with:
-          key: ubuntu-latest-sanitizer-${{ matrix.sanitizer }}
-          evict-old-files: 1d
-          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+      #- name: ccache
+      #  uses: ggml-org/ccache-action@v1.2.21
+      #  with:
+      #    key: ubuntu-latest-sanitizer-${{ matrix.sanitizer }}
+      #    evict-old-files: 1d
+      #    save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 
-      - name: Dependencies
-        id: depends
+      #- name: Dependencies
+      #  id: depends
+      #  run: |
+      #    sudo apt-get update
+      #    sudo apt-get install build-essential libssl-dev
+
+      # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
+      - name: Build (undefined)
+        id: cmake_build_undefined
+        if: ${{ matrix.sanitizer == 'UNDEFINED' }}
         run: |
-          sudo apt-get update
-          sudo apt-get install build-essential libssl-dev
+          cmake -B build \
+            -DLLAMA_FATAL_WARNINGS=ON \
+            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
+            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
+
+          cmake --build build --config Debug -j $(nproc)
 
       - name: Build
         id: cmake_build
         if: ${{ matrix.sanitizer != 'THREAD' }}
         run: |
           cmake -B build \
-            -DLLAMA_FATAL_WARNINGS=ON \
             -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
-            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
-            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
+            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
 
-          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+          cmake --build build --config RelWithDebInfo -j $(nproc)
 
       - name: Build (no OpenMP)
         id: cmake_build_no_openmp
         if: ${{ matrix.sanitizer == 'THREAD' }}
         run: |
           cmake -B build \
-            -DLLAMA_FATAL_WARNINGS=ON \
             -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
             -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
-            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
             -DGGML_OPENMP=OFF
 
-          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
+          cmake --build build --config RelWithDebInfo -j $(nproc)
 
       - name: Test
         id: cmake_test
+        # skip run in Debug - very slow
+        if: ${{ matrix.sanitizer != 'UNDEFINED' }}
         run: |
           cd build
-          ctest -L main --verbose --timeout 900
+          ctest -L main -E tokenizer --verbose --timeout 900

diff --git a/.github/workflows/server-sanitize.yml b/.github/workflows/server-sanitize.yml
index 53c9968ee967f0d902d79860950673a7f1d1df4a..46e47467210d69e26a8f9e3f9ee0dc767baaf875 100644 (file)
--- a/.github/workflows/server-sanitize.yml
+++ b/.github/workflows/server-sanitize.yml
@@ -37,7 +37,7 @@ concurrency:
 
 jobs:
   server:
-    runs-on: ubuntu-latest
+    runs-on: [self-hosted, CPU, Linux, llama-server]
 
     strategy:
       matrix:
@@ -46,19 +46,19 @@ jobs:
       fail-fast: false
 
     steps:
-      - name: Dependencies
-        id: depends
-        run: |
-          sudo apt-get update
-          sudo apt-get -y install \
-            build-essential \
-            xxd \
-            git \
-            cmake \
-            curl \
-            wget \
-            language-pack-en \
-            libssl-dev
+      #- name: Dependencies
+      #  id: depends
+      #  run: |
+      #    sudo apt-get update
+      #    sudo apt-get -y install \
+      #      build-essential \
+      #      xxd \
+      #      git \
+      #      cmake \
+      #      curl \
+      #      wget \
+      #      language-pack-en \
+      #      libssl-dev
 
       - name: Clone
         id: checkout

diff --git a/tests/gguf-model-data.cpp b/tests/gguf-model-data.cpp
index d277173c4666fe2ad74c4d8e8dcc683c1d76b609..ba3aa76f75213a5ac4aee6fd13c19404634b93d9 100644 (file)
--- a/tests/gguf-model-data.cpp
+++ b/tests/gguf-model-data.cpp
@@ -630,10 +630,11 @@ std::optional<gguf_remote_model> gguf_fetch_model_meta(
         }
 
         for (int i = 2; i <= model.n_split; i++) {
-            char num_buf[6], total_buf[6];
-            snprintf(num_buf,   sizeof(num_buf),   "%05d", i);
-            snprintf(total_buf, sizeof(total_buf), "%05d", (int)model.n_split);
-            std::string shard_name = split_prefix + "-" + num_buf + "-of-" + total_buf + ".gguf";
+            char buf_num[32];
+            char buf_tot[32];
+            snprintf(buf_num, sizeof(buf_num), "%05d", i);
+            snprintf(buf_tot, sizeof(buf_tot), "%05d", (int)model.n_split);
+            std::string shard_name = split_prefix + "-" + buf_num + "-of-" + buf_tot + ".gguf";
 
             auto shard = fetch_or_cached(repo, shard_name, cdir, repo_part, verbose);
             if (!shard.has_value()) {