]> git.djapps.eu Git - pkg/ggml/sources/ggml/commit
overview / pkg / ggml / sources / ggml / commit
summary | shortlog | log | commit | commitdiff | tree
(parent: 174189e)
CUDA: quantized KV support for FA vec (llama/7527)
authorJohannes Gäßler <redacted>
Sat, 1 Jun 2024 06:44:14 +0000 (08:44 +0200)
committerGeorgi Gerganov <redacted>
Sat, 15 Jun 2024 19:05:47 +0000 (22:05 +0300)
commitebe78772cd321ed1c969e58da951b0ff72b13110
tree14674041f90f48f152acd642f29bf02f1d31e5batree
parent174189ec5d702a7b698dc6a83c3461a52a04b117commit | diff
CUDA: quantized KV support for FA vec (llama/7527)

* CUDA: quantized KV support for FA vec

* try CI fix

* fix commented-out kernel variants

* add q8_0 q4_0 tests

* fix nwarps > batch size

* split fattn compile via extern templates

* fix flake8

* fix metal tests

* fix cmake

* make generate_cu_files.py executable

* add autogenerated .cu files

* fix AMD

* error if type_v != FP16 and not flash_attn

* remove obsolete code
104 files changed:
src/ggml-cuda.cu diff | blob | history
src/ggml-cuda/fattn-common.cuh diff | blob | history
src/ggml-cuda/fattn-tile-f16.cu diff | blob | history
src/ggml-cuda/fattn-tile-f32.cu diff | blob | history
src/ggml-cuda/fattn-vec-f16.cuh diff | blob | history
src/ggml-cuda/fattn-vec-f32.cuh diff | blob | history
src/ggml-cuda/fattn-wmma-f16.cuh [new file with mode: 0644] blob
src/ggml-cuda/fattn.cu diff | blob | history
src/ggml-cuda/mmq.cu diff | blob | history
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_1-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q5_1-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_1-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q5_1-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q4_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q5_1.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-q8_0.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.cu [new file with mode: 0644] blob
src/ggml-cuda/template-instances/generate_cu_files.py [new file with mode: 0755] blob
src/ggml-cuda/vecdotq.cuh diff | blob | history
src/ggml-metal.m diff | blob | history
tests/test-backend-ops.cpp diff | blob | history
Packaging of ggml-org/ggml