include(CheckCXXCompilerFlag)
-unset(GGML_CDEF_PUBLIC)
-
add_compile_definitions(GGML_SCHED_MAX_COPIES=${GGML_SCHED_MAX_COPIES})
# enable libstdc++ assertions for debug builds
endif()
endif()
-unset(GGML_EXTRA_LIBS_PRIVATE)
-unset(GGML_EXTRA_LIBS_PUBLIC)
-
-if (APPLE AND GGML_ACCELERATE)
- find_library(ACCELERATE_FRAMEWORK Accelerate)
- if (ACCELERATE_FRAMEWORK)
- message(STATUS "Accelerate framework found")
-
- add_compile_definitions(GGML_USE_ACCELERATE)
- add_compile_definitions(ACCELERATE_NEW_LAPACK)
- add_compile_definitions(ACCELERATE_LAPACK_ILP64)
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE ${ACCELERATE_FRAMEWORK})
- else()
- message(WARNING "Accelerate framework not found")
- endif()
-endif()
-
-if (GGML_METAL)
- find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
- find_library(METAL_FRAMEWORK Metal REQUIRED)
- find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
-
- message(STATUS "Metal framework found")
- set(GGML_HEADERS_METAL ../include/ggml-metal.h)
- set(GGML_SOURCES_METAL ggml-metal.m)
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_METAL)
- if (GGML_METAL_NDEBUG)
- add_compile_definitions(GGML_METAL_NDEBUG)
- endif()
-
- if (GGML_METAL_USE_BF16)
- add_compile_definitions(GGML_METAL_USE_BF16)
- endif()
-
- # copy ggml-common.h and ggml-metal.metal to bin directory
- configure_file(ggml-common.h ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h COPYONLY)
- configure_file(ggml-metal.metal ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
-
- if (GGML_METAL_EMBED_LIBRARY)
- enable_language(ASM)
-
- add_compile_definitions(GGML_METAL_EMBED_LIBRARY)
-
- set(METALLIB_COMMON "${CMAKE_CURRENT_SOURCE_DIR}/ggml-common.h")
- set(METALLIB_SOURCE "${CMAKE_CURRENT_SOURCE_DIR}/ggml-metal.metal")
-
- file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/autogenerated")
-
- # merge ggml-common.h and ggml-metal.metal into a single file
- set(METALLIB_EMBED_ASM "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.s")
- set(METALLIB_SOURCE_EMBED "${CMAKE_BINARY_DIR}/autogenerated/ggml-metal-embed.metal")
-
- add_custom_command(
- OUTPUT ${METALLIB_EMBED_ASM}
- COMMAND echo "Embedding Metal library"
- COMMAND sed -e '/\#include \"ggml-common.h\"/r ${METALLIB_COMMON}' -e '/\#include \"ggml-common.h\"/d' < ${METALLIB_SOURCE} > ${METALLIB_SOURCE_EMBED}
- COMMAND echo ".section __DATA,__ggml_metallib" > ${METALLIB_EMBED_ASM}
- COMMAND echo ".globl _ggml_metallib_start" >> ${METALLIB_EMBED_ASM}
- COMMAND echo "_ggml_metallib_start:" >> ${METALLIB_EMBED_ASM}
- COMMAND echo ".incbin \\\"${METALLIB_SOURCE_EMBED}\\\"" >> ${METALLIB_EMBED_ASM}
- COMMAND echo ".globl _ggml_metallib_end" >> ${METALLIB_EMBED_ASM}
- COMMAND echo "_ggml_metallib_end:" >> ${METALLIB_EMBED_ASM}
- DEPENDS ggml-metal.metal ggml-common.h
- COMMENT "Generate assembly for embedded Metal library"
- )
-
- list(APPEND GGML_SOURCES_METAL ${METALLIB_EMBED_ASM})
- else()
- if (GGML_METAL_SHADER_DEBUG)
- # custom command to do the following:
- # xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air
- # xcrun -sdk macosx metallib ggml-metal.air -o default.metallib
- #
- # note: this is the only way I found to disable fast-math in Metal. it's ugly, but at least it works
- # disabling fast math is needed in order to pass tests/test-backend-ops
- # note: adding -fno-inline fixes the tests when using MTL_SHADER_VALIDATION=1
- # note: unfortunately, we have to call it default.metallib instead of ggml.metallib
- # ref: https://github.com/ggerganov/whisper.cpp/issues/1720
- set(XC_FLAGS -fno-fast-math -fno-inline -g)
- else()
- set(XC_FLAGS -O3)
- endif()
-
- # Append macOS metal versioning flags
- if (GGML_METAL_MACOSX_VERSION_MIN)
- message(STATUS "Adding -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN} flag to metal compilation")
- list (APPEND XC_FLAGS -mmacosx-version-min=${GGML_METAL_MACOSX_VERSION_MIN})
- endif()
-
- if (GGML_METAL_STD)
- message(STATUS "Adding -std=${GGML_METAL_STD} flag to metal compilation")
- list (APPEND XC_FLAGS -std=${GGML_METAL_STD})
- endif()
-
- add_custom_command(
- OUTPUT ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
- COMMAND xcrun -sdk macosx metal ${XC_FLAGS} -c ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
- COMMAND xcrun -sdk macosx metallib ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air -o ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
- COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.air
- COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-common.h
- COMMAND rm -f ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/ggml-metal.metal
- DEPENDS ggml-metal.metal ggml-common.h
- COMMENT "Compiling Metal kernels"
- )
-
- add_custom_target(
- ggml-metal ALL
- DEPENDS ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/default.metallib
- )
- endif() # GGML_METAL_EMBED_LIBRARY
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE
- ${FOUNDATION_LIBRARY}
- ${METAL_FRAMEWORK}
- ${METALKIT_FRAMEWORK}
- )
-endif()
-
-if (GGML_MUSA)
- set(CMAKE_C_COMPILER clang)
- set(CMAKE_C_EXTENSIONS OFF)
- set(CMAKE_CXX_COMPILER clang++)
- set(CMAKE_CXX_EXTENSIONS OFF)
-
- set(GGML_CUDA ON)
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_MUSA)
-endif()
-
-if (GGML_OPENMP)
- find_package(OpenMP)
- if (OpenMP_FOUND)
- message(STATUS "OpenMP found")
-
- add_compile_definitions(GGML_USE_OPENMP)
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE OpenMP::OpenMP_C OpenMP::OpenMP_CXX)
-
- if (GGML_MUSA)
- list(APPEND GGML_EXTRA_INCLUDES "/usr/lib/llvm-14/lib/clang/14.0.0/include")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE "/usr/lib/llvm-14/lib/libomp.so")
- endif()
- else()
- message(WARNING "OpenMP not found")
- endif()
-endif()
-
-if (GGML_BLAS)
- if (GGML_STATIC)
- set(BLA_STATIC ON)
- endif()
- #if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
- # set(BLA_SIZEOF_INTEGER 8)
- #endif()
-
- set(BLA_VENDOR ${GGML_BLAS_VENDOR})
- find_package(BLAS)
-
- if (BLAS_FOUND)
- message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
-
- if (("${BLAS_INCLUDE_DIRS}" STREQUAL "") AND NOT (${GGML_BLAS_VENDOR} MATCHES "Apple"))
- # BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
- # see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
- find_package(PkgConfig REQUIRED)
- if (${GGML_BLAS_VENDOR} MATCHES "Generic")
- pkg_check_modules(DepBLAS blas)
- elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
- # As of openblas v0.3.22, the 64-bit is named openblas64.pc
- pkg_check_modules(DepBLAS openblas64)
- if (NOT DepBLAS_FOUND)
- pkg_check_modules(DepBLAS openblas)
- endif()
- elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
- add_compile_definitions(GGML_BLAS_USE_BLIS)
- pkg_check_modules(DepBLAS blis)
- elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
- pkg_check_modules(DepBLAS blas-atlas)
- elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
- pkg_check_modules(DepBLAS flexiblas_api)
- elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
- add_compile_definitions(GGML_BLAS_USE_MKL)
- # all Intel* libraries share the same include path
- pkg_check_modules(DepBLAS mkl-sdl)
- elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
- # this doesn't provide pkg-config
- # suggest to assign BLAS_INCLUDE_DIRS on your own
- if ("${NVHPC_VERSION}" STREQUAL "")
- message(WARNING "Better to set NVHPC_VERSION")
- else()
- set(DepBLAS_FOUND ON)
- set(DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include")
- endif()
- endif()
- if (DepBLAS_FOUND)
- set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})
- else()
- message(WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
- " detected by pkgconfig, trying to find cblas.h from possible paths...")
- find_path(BLAS_INCLUDE_DIRS
- NAMES cblas.h
- HINTS
- /usr/include
- /usr/local/include
- /usr/include/openblas
- /opt/homebrew/opt/openblas/include
- /usr/local/opt/openblas/include
- /usr/include/x86_64-linux-gnu/openblas/include
- )
- endif()
- endif()
-
- message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}")
-
- add_compile_options(${BLAS_LINKER_FLAGS})
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_BLAS)
-
- if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
- add_compile_definitions(GGML_BLAS_USE_MKL)
- endif()
-
- set(GGML_HEADERS_BLAS ../include/ggml-blas.h)
- set(GGML_SOURCES_BLAS ggml-blas.cpp)
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE ${BLAS_LIBRARIES})
- list(APPEND GGML_EXTRA_INCLUDES ${BLAS_INCLUDE_DIRS})
- else()
- message(WARNING "BLAS not found, please refer to "
- "https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
- " to set correct GGML_BLAS_VENDOR")
- endif()
-endif()
-
-if (GGML_LLAMAFILE)
- message(STATUS "Using llamafile")
-
- add_compile_definitions(GGML_USE_LLAMAFILE)
-
- set(GGML_HEADERS_LLAMAFILE llamafile/sgemm.h)
- set(GGML_SOURCES_LLAMAFILE llamafile/sgemm.cpp)
-endif()
-
-if (GGML_AMX)
- if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.0)
- else()
- set(GGML_AMX OFF)
- message(WARNING "AMX requires gcc version > 11.0. Turning off GGML_AMX.")
- endif()
-
- if (GGML_AMX)
- message(STATUS "Using AMX")
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_AMX)
-
- file(GLOB GGML_HEADERS_AMX "ggml-amx/*.h")
- list(APPEND GGML_HEADERS_AMX "../include/ggml-amx.h")
-
- file(GLOB GGML_SOURCES_AMX "ggml-amx/*.cpp")
- list(APPEND GGML_SOURCES_AMX "ggml-amx.cpp")
- endif()
-endif()
-
-if (GGML_CUDA)
- cmake_minimum_required(VERSION 3.18) # for CMAKE_CUDA_ARCHITECTURES
-
- if (GGML_MUSA)
- list(APPEND CMAKE_MODULE_PATH "/usr/local/musa/cmake/")
- find_package(MUSAToolkit)
- set(CUDAToolkit_FOUND ${MUSAToolkit_FOUND})
- else()
- find_package(CUDAToolkit)
- endif()
-
- if (CUDAToolkit_FOUND)
- message(STATUS "CUDA found")
-
- if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
- # 52 == lowest CUDA 12 standard
- # 60 == FP16 CUDA intrinsics
- # 61 == integer CUDA intrinsics
- # 70 == compute capability at which unrolling a loop in mul_mat_q kernels is faster
- if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
- set(CMAKE_CUDA_ARCHITECTURES "60;61;70;75")
- else()
- set(CMAKE_CUDA_ARCHITECTURES "52;61;70;75")
- #set(CMAKE_CUDA_ARCHITECTURES "OFF") # use this to compile much faster, but only F16 models work
- endif()
- endif()
- message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}")
-
- if (GGML_MUSA)
- set(CMAKE_CUDA_COMPILER ${MUSAToolkit_MCC_EXECUTABLE})
- else()
- enable_language(CUDA)
- endif()
-
- file(GLOB GGML_HEADERS_CUDA "ggml-cuda/*.cuh")
- list(APPEND GGML_HEADERS_CUDA "../include/ggml-cuda.h")
-
- file(GLOB GGML_SOURCES_CUDA "ggml-cuda/*.cu")
- list(APPEND GGML_SOURCES_CUDA "ggml-cuda.cu")
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-wmma*.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/mmq*.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
-
- if (GGML_CUDA_FA_ALL_QUANTS)
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
- add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)
- else()
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*f16-f16.cu")
- list(APPEND GGML_SOURCES_CUDA ${SRCS})
- endif()
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_CUDA)
-
- add_compile_definitions(GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
- add_compile_definitions(GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
- add_compile_definitions(K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
- add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
-
- if (GGML_CUDA_GRAPHS)
- add_compile_definitions(GGML_CUDA_USE_GRAPHS)
- endif()
-
- if (GGML_CUDA_FORCE_DMMV)
- add_compile_definitions(GGML_CUDA_FORCE_DMMV)
- endif()
-
- if (GGML_CUDA_FORCE_MMQ)
- add_compile_definitions(GGML_CUDA_FORCE_MMQ)
- endif()
-
- if (GGML_CUDA_FORCE_CUBLAS)
- add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)
- endif()
-
- if (GGML_CUDA_NO_VMM)
- add_compile_definitions(GGML_CUDA_NO_VMM)
- endif()
-
- if (DEFINED GGML_CUDA_DMMV_Y)
- add_compile_definitions(GGML_CUDA_MMV_Y=${GGML_CUDA_DMMV_Y}) # for backwards compatibility
- endif()
-
- if (GGML_CUDA_F16 OR GGML_CUDA_DMMV_F16)
- add_compile_definitions(GGML_CUDA_F16)
- endif()
-
- if (GGML_CUDA_NO_PEER_COPY)
- add_compile_definitions(GGML_CUDA_NO_PEER_COPY)
- endif()
-
- if (GGML_MUSA)
- set_source_files_properties(${GGML_SOURCES_CUDA} PROPERTIES LANGUAGE CXX)
- foreach(SOURCE ${GGML_SOURCES_CUDA})
- set_property(SOURCE ${SOURCE} PROPERTY COMPILE_FLAGS "-x musa -mtgpu --cuda-gpu-arch=mp_21 --cuda-gpu-arch=mp_22")
- endforeach()
- endif()
-
- if (GGML_STATIC)
- if (WIN32)
- # As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library
- list(APPEND GGML_EXTRA_LIBS_PRIVATE CUDA::cudart_static CUDA::cublas CUDA::cublasLt)
- else ()
- if (GGML_MUSA)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE MUSA::musart_static MUSA::mublas_static)
- else()
- list(APPEND GGML_EXTRA_LIBS_PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
- endif()
- endif()
- else()
- if (GGML_MUSA)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE MUSA::musart MUSA::mublas)
- else()
- list(APPEND GGML_EXTRA_LIBS_PRIVATE CUDA::cudart CUDA::cublas CUDA::cublasLt)
- endif()
- endif()
-
- if (GGML_CUDA_NO_VMM)
- # No VMM requested, no need to link directly with the cuda driver lib (libcuda.so)
- else()
- if (GGML_MUSA)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE MUSA::musa_driver) # required by muDeviceGetAttribute(), muMemGetAllocationGranularity(...), ...
- else()
- list(APPEND GGML_EXTRA_LIBS_PRIVATE CUDA::cuda_driver) # required by cuDeviceGetAttribute(), cuMemGetAllocationGranularity(...), ...
- endif()
- endif()
- else()
- message(WARNING "CUDA not found")
- endif()
-endif()
-
-if (GGML_HIPBLAS)
- if (NOT EXISTS $ENV{ROCM_PATH})
- if (NOT EXISTS /opt/rocm)
- set(ROCM_PATH /usr)
- else()
- set(ROCM_PATH /opt/rocm)
- endif()
- else()
- set(ROCM_PATH $ENV{ROCM_PATH})
- endif()
-
- list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH})
- list(APPEND CMAKE_PREFIX_PATH "${ROCM_PATH}/lib64/cmake")
-
- # CMake on Windows doesn't support the HIP language yet
- if (WIN32)
- set(CXX_IS_HIPCC TRUE)
- else()
- string(REGEX MATCH "hipcc(\.bat)?$" CXX_IS_HIPCC "${CMAKE_CXX_COMPILER}")
- endif()
-
- if (CXX_IS_HIPCC)
- if (LINUX)
- if (NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
- message(WARNING "Only LLVM is supported for HIP, hint: CXX=/opt/rocm/llvm/bin/clang++")
- endif()
-
- message(WARNING "Setting hipcc as the C++ compiler is legacy behavior."
- " Prefer setting the HIP compiler directly. See README for details.")
- endif()
- else()
- # Forward AMDGPU_TARGETS to CMAKE_HIP_ARCHITECTURES.
- if (AMDGPU_TARGETS AND NOT CMAKE_HIP_ARCHITECTURES)
- set(CMAKE_HIP_ARCHITECTURES ${AMDGPU_TARGETS})
- endif()
- cmake_minimum_required(VERSION 3.21)
- enable_language(HIP)
- endif()
-
- find_package(hip REQUIRED)
- find_package(hipblas REQUIRED)
- find_package(rocblas REQUIRED)
-
- message(STATUS "HIP and hipBLAS found")
-
- file(GLOB GGML_HEADERS_ROCM "ggml-cuda/*.cuh")
- list(APPEND GGML_HEADERS_ROCM "../include/ggml-cuda.h")
-
- file(GLOB GGML_SOURCES_ROCM "ggml-cuda/*.cu")
- list(APPEND GGML_SOURCES_ROCM "ggml-cuda.cu")
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-wmma*.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/mmq*.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
-
- if (GGML_CUDA_FA_ALL_QUANTS)
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
- add_compile_definitions(GGML_CUDA_FA_ALL_QUANTS)
- else()
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*q4_0-q4_0.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*q8_0-q8_0.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
- file(GLOB SRCS "ggml-cuda/template-instances/fattn-vec*f16-f16.cu")
- list(APPEND GGML_SOURCES_ROCM ${SRCS})
- endif()
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_CUDA)
-
- add_compile_definitions(GGML_USE_HIPBLAS)
- add_compile_definitions(GGML_CUDA_DMMV_X=${GGML_CUDA_DMMV_X})
- add_compile_definitions(GGML_CUDA_MMV_Y=${GGML_CUDA_MMV_Y})
- add_compile_definitions(K_QUANTS_PER_ITERATION=${GGML_CUDA_KQUANTS_ITER})
-
- if (GGML_HIP_UMA)
- add_compile_definitions(GGML_HIP_UMA)
- endif()
-
- if (GGML_CUDA_FORCE_DMMV)
- add_compile_definitions(GGML_CUDA_FORCE_DMMV)
- endif()
-
- if (GGML_CUDA_FORCE_MMQ)
- add_compile_definitions(GGML_CUDA_FORCE_MMQ)
- endif()
-
- if (GGML_CUDA_FORCE_CUBLAS)
- add_compile_definitions(GGML_CUDA_FORCE_CUBLAS)
- endif()
-
- if (GGML_CUDA_NO_PEER_COPY)
- add_compile_definitions(GGML_CUDA_NO_PEER_COPY)
- endif()
-
- if (CXX_IS_HIPCC)
- set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE CXX)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE hip::device)
- else()
- set_source_files_properties(${GGML_SOURCES_ROCM} PROPERTIES LANGUAGE HIP)
- endif()
-
- if (GGML_STATIC)
- message(FATAL_ERROR "Static linking not supported for HIP/ROCm")
- endif()
-
- list(APPEND GGML_EXTRA_LIBS_PUBLIC hip::host roc::rocblas roc::hipblas)
-endif()
-
-if (GGML_SYCL)
- if (NOT GGML_SYCL_TARGET MATCHES "^(INTEL|NVIDIA|AMD)$")
- message(FATAL_ERROR "Invalid backend chosen, supported options are INTEL, NVIDIA, or AMD")
- endif()
-
- check_cxx_compiler_flag("-fsycl" SUPPORTS_SYCL)
-
- if (DEFINED ENV{ONEAPI_ROOT})
- message(STATUS "Using oneAPI Release SYCL compiler (icpx).")
- elseif(SUPPORTS_SYCL)
- message(WARNING "Using open-source SYCL compiler (clang++). Didn't detect ENV {ONEAPI_ROOT}.
- If you expected the oneAPI Release compiler, please install oneAPI & source it, like:
- source /opt/intel/oneapi/setvars.sh")
- else()
- message(FATAL_ERROR, "C++ compiler lacks SYCL support.")
- endif()
- message(STATUS "SYCL found")
- #todo: AOT
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_SYCL)
-
- if (GGML_SYCL_F16)
- if (GGML_SYCL_TARGET STREQUAL "AMD")
- message(WARNING "AMD target does not entirely support FP16 in the SYCL backend.")
- endif()
- add_compile_definitions(GGML_SYCL_F16)
- endif()
-
- if (GGML_CUDA_FORCE_MMQ)
- add_compile_definitions(GGML_SYCL_FORCE_MMQ)
- endif()
-
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing -fsycl")
-
- if (GGML_SYCL_TARGET STREQUAL "NVIDIA")
- add_compile_definitions(GGML_SYCL_WARP_SIZE=32)
- elseif (GGML_SYCL_TARGET STREQUAL "AMD")
- # INFO: Allowed Sub_group_sizes are not consistent through all
- # hip targets. For example, 64 is used for certain models, but the backend
- # does not support it.
- # Target archs tested working: gfx1030, gfx1031, (Only tested sub_group_size = 32)
- add_compile_definitions(GGML_SYCL_WARP_SIZE=32)
- else()
- add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
- endif()
-
- file(GLOB GGML_HEADERS_SYCL "ggml-sycl/*.hpp")
- list(APPEND GGML_HEADERS_SYCL "../include/ggml-sycl.h")
-
- file(GLOB GGML_SOURCES_SYCL "ggml-sycl/*.cpp")
- list(APPEND GGML_SOURCES_SYCL "ggml-sycl.cpp")
-
- find_package(DNNL)
- message("-- DNNL found:" ${DNNL_FOUND})
-
- if (GGML_SYCL_TARGET STREQUAL "INTEL")
- add_compile_definitions(GGML_SYCL_DNNL=${DNNL_FOUND})
- else()
- add_compile_definitions(GGML_SYCL_DNNL=0)
- endif()
-
- if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE DNNL::dnnl)
- endif()
-
- if (WIN32)
- find_package(IntelSYCL REQUIRED)
- find_package(MKL REQUIRED)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
- else()
- if (GGML_SYCL_TARGET STREQUAL "INTEL")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread)
- elseif (GGML_SYCL_TARGET STREQUAL "NVIDIA")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=nvptx64-nvidia-cuda")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl pthread m dl onemkl)
- elseif (GGML_SYCL_TARGET STREQUAL "AMD")
- if (GGML_SYCL_HIP_TARGET STREQUAL "")
- message(ERROR "Can't enable SYCL hip backend, GGML_SYCL_HIP_TARGET has not been set.")
- endif()
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=${GGML_SYCL_HIP_TARGET}")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl pthread m dl onemkl)
- endif()
- endif()
-endif()
-
-if (GGML_RPC)
- message(STATUS "RPC found")
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_RPC)
-
- if (WIN32)
- list(APPEND GGML_EXTRA_LIBS_PRIVATE ws2_32)
- endif()
-
- set(GGML_HEADERS_RPC ../include/ggml-rpc.h)
- set(GGML_SOURCES_RPC ggml-rpc.cpp)
-endif()
-
-if (GGML_VULKAN)
- find_package(Vulkan COMPONENTS glslc REQUIRED)
-
- if (Vulkan_FOUND)
- message(STATUS "Vulkan found")
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_VULKAN)
-
- # Workaround to the "can't dereference invalidated vector iterator" bug in clang-cl debug build
- # Posssibly relevant: https://stackoverflow.com/questions/74748276/visual-studio-no-displays-the-correct-length-of-stdvector
- if (MSVC AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
- add_compile_definitions(_ITERATOR_DEBUG_LEVEL=0)
- endif()
-
- if (GGML_VULKAN_CHECK_RESULTS)
- add_compile_definitions(GGML_VULKAN_CHECK_RESULTS)
- endif()
-
- if (GGML_VULKAN_DEBUG)
- add_compile_definitions(GGML_VULKAN_DEBUG)
- endif()
-
- if (GGML_VULKAN_MEMORY_DEBUG)
- add_compile_definitions(GGML_VULKAN_MEMORY_DEBUG)
- endif()
-
- if (GGML_VULKAN_SHADER_DEBUG_INFO)
- add_compile_definitions(GGML_VULKAN_SHADER_DEBUG_INFO)
- endif()
-
- if (GGML_VULKAN_PERF)
- add_compile_definitions(GGML_VULKAN_PERF)
- endif()
-
- if (GGML_VULKAN_VALIDATE)
- add_compile_definitions(GGML_VULKAN_VALIDATE)
- endif()
-
- if (GGML_VULKAN_RUN_TESTS)
- add_compile_definitions(GGML_VULKAN_RUN_TESTS)
- endif()
-
- add_subdirectory(vulkan-shaders)
-
- set (_ggml_vk_genshaders_cmd vulkan-shaders-gen)
- set (_ggml_vk_header ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan-shaders.hpp)
- set (_ggml_vk_source ${CMAKE_CURRENT_BINARY_DIR}/ggml-vulkan-shaders.cpp)
- set (_ggml_vk_input_dir ${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders)
- set (_ggml_vk_output_dir ${CMAKE_CURRENT_BINARY_DIR}/vulkan-shaders.spv)
-
- file(GLOB _ggml_vk_shader_deps "${_ggml_vk_input_dir}/*.comp")
-
- add_custom_command(
- OUTPUT ${_ggml_vk_header}
- ${_ggml_vk_source}
-
- COMMAND ${_ggml_vk_genshaders_cmd}
- --glslc ${Vulkan_GLSLC_EXECUTABLE}
- --input-dir ${_ggml_vk_input_dir}
- --output-dir ${_ggml_vk_output_dir}
- --target-hpp ${_ggml_vk_header}
- --target-cpp ${_ggml_vk_source}
- --no-clean
-
- DEPENDS ${_ggml_vk_shader_deps}
- COMMENT "Generate vulkan shaders"
- )
-
- set(GGML_HEADERS_VULKAN ${CMAKE_CURRENT_SOURCE_DIR}/../include/ggml-vulkan.h ${_ggml_vk_header})
- set(GGML_SOURCES_VULKAN ggml-vulkan.cpp ${_ggml_vk_source})
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE Vulkan::Vulkan)
- list(APPEND GGML_EXTRA_INCLUDES ${CMAKE_CURRENT_BINARY_DIR})
- else()
- message(WARNING "Vulkan not found")
- endif()
-endif()
-
-if (GGML_KOMPUTE)
- add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
-
- find_package(Vulkan COMPONENTS glslc REQUIRED)
- find_program(glslc_executable NAMES glslc HINTS Vulkan::glslc)
-
- if (NOT glslc_executable)
- message(FATAL_ERROR "glslc not found")
- endif()
-
- function(compile_shader)
- set(options)
- set(oneValueArgs)
- set(multiValueArgs SOURCES)
- cmake_parse_arguments(compile_shader "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
- foreach(source ${compile_shader_SOURCES})
- get_filename_component(filename ${source} NAME)
- set(spv_file ${filename}.spv)
- add_custom_command(
- OUTPUT ${spv_file}
- DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${source}
- ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/common.comp
- ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_getrows.comp
- ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n_pre.comp
- ${CMAKE_CURRENT_SOURCE_DIR}/kompute-shaders/op_mul_mv_q_n.comp
- COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR}/${source}
- COMMENT "Compiling ${source} to ${spv_file}"
- )
-
- get_filename_component(RAW_FILE_NAME ${spv_file} NAME)
- set(FILE_NAME "shader${RAW_FILE_NAME}")
- string(REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME})
- string(TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE)
- string(REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE}")
- set(OUTPUT_HEADER_FILE "${HEADER_FILE}")
- message(STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE}")
- if(CMAKE_GENERATOR MATCHES "Visual Studio")
- add_custom_command(
- OUTPUT ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- DEPENDS ${spv_file} xxd
- COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/$<CONFIG>/xxd"
- )
- else()
- add_custom_command(
- OUTPUT ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#ifndef ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "namespace kp {" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "namespace shader_data {" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_BINARY_DIR}/bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo "}}" >> ${OUTPUT_HEADER_FILE}
- COMMAND ${CMAKE_COMMAND} -E echo \"\#endif // define ${HEADER_FILE_DEFINE}\" >> ${OUTPUT_HEADER_FILE}
- DEPENDS ${spv_file} xxd
- COMMENT "Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR}/bin/xxd"
- )
- endif()
- endforeach()
- endfunction()
-
- if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/kompute/CMakeLists.txt")
- message(STATUS "Kompute found")
- set(KOMPUTE_OPT_LOG_LEVEL Error CACHE STRING "Kompute log level")
- add_subdirectory(kompute)
-
- # Compile our shaders
- compile_shader(SOURCES
- kompute-shaders/op_scale.comp
- kompute-shaders/op_scale_8.comp
- kompute-shaders/op_add.comp
- kompute-shaders/op_addrow.comp
- kompute-shaders/op_mul.comp
- kompute-shaders/op_silu.comp
- kompute-shaders/op_relu.comp
- kompute-shaders/op_gelu.comp
- kompute-shaders/op_softmax.comp
- kompute-shaders/op_norm.comp
- kompute-shaders/op_rmsnorm.comp
- kompute-shaders/op_diagmask.comp
- kompute-shaders/op_mul_mat_mat_f32.comp
- kompute-shaders/op_mul_mat_f16.comp
- kompute-shaders/op_mul_mat_q8_0.comp
- kompute-shaders/op_mul_mat_q4_0.comp
- kompute-shaders/op_mul_mat_q4_1.comp
- kompute-shaders/op_mul_mat_q4_k.comp
- kompute-shaders/op_mul_mat_q6_k.comp
- kompute-shaders/op_getrows_f32.comp
- kompute-shaders/op_getrows_f16.comp
- kompute-shaders/op_getrows_q4_0.comp
- kompute-shaders/op_getrows_q4_1.comp
- kompute-shaders/op_getrows_q6_k.comp
- kompute-shaders/op_rope_f16.comp
- kompute-shaders/op_rope_f32.comp
- kompute-shaders/op_cpy_f16_f16.comp
- kompute-shaders/op_cpy_f16_f32.comp
- kompute-shaders/op_cpy_f32_f16.comp
- kompute-shaders/op_cpy_f32_f32.comp
- )
-
- # Create a custom target for our generated shaders
- add_custom_target(generated_shaders DEPENDS
- shaderop_scale.h
- shaderop_scale_8.h
- shaderop_add.h
- shaderop_addrow.h
- shaderop_mul.h
- shaderop_silu.h
- shaderop_relu.h
- shaderop_gelu.h
- shaderop_softmax.h
- shaderop_norm.h
- shaderop_rmsnorm.h
- shaderop_diagmask.h
- shaderop_mul_mat_mat_f32.h
- shaderop_mul_mat_f16.h
- shaderop_mul_mat_q8_0.h
- shaderop_mul_mat_q4_0.h
- shaderop_mul_mat_q4_1.h
- shaderop_mul_mat_q4_k.h
- shaderop_mul_mat_q6_k.h
- shaderop_getrows_f32.h
- shaderop_getrows_f16.h
- shaderop_getrows_q4_0.h
- shaderop_getrows_q4_1.h
- shaderop_getrows_q6_k.h
- shaderop_rope_f16.h
- shaderop_rope_f32.h
- shaderop_cpy_f16_f16.h
- shaderop_cpy_f16_f32.h
- shaderop_cpy_f32_f16.h
- shaderop_cpy_f32_f32.h
- )
-
- # Create a custom command that depends on the generated_shaders
- add_custom_command(
- OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
- COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp
- DEPENDS generated_shaders
- COMMENT "Ensuring shaders are generated before compiling ggml-kompute.cpp"
- )
-
- # Add the stamp to the main sources to ensure dependency tracking
- set(GGML_SOURCES_KOMPUTE ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
- set(GGML_HEADERS_KOMPUTE ../include/ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR}/ggml-kompute.stamp)
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_KOMPUTE)
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE kompute)
- list(APPEND GGML_EXTRA_INCLUDES ${CMAKE_CURRENT_BINARY_DIR})
- else()
- message(WARNING "Kompute not found")
- endif()
-endif()
-
-if (GGML_CPU_HBM)
- find_library(memkind memkind REQUIRED)
-
- message(STATUS "Using memkind for CPU HBM")
-
- add_compile_definitions(GGML_USE_CPU_HBM)
-
- target_link_libraries(ggml PUBLIC memkind)
-endif()
-
-if (GGML_CANN)
- if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})
- set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})
- message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
- endif()
-
- if (CANN_INSTALL_DIR)
- # Only Support Linux.
- if (GGML_CANN)
- if (NOT UNIX)
- set(GGML_CANN OFF)
- message(WARNING "CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}. Turning off GGML_CANN")
- endif()
- endif()
-
- # Supported platforms: x86-64, arm64
- if (GGML_CANN)
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
- elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- else()
- set(GGML_CANN OFF)
- message(WARNING "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off GGML_CANN")
- endif()
- endif()
-
- # Set header and libs
- if(GGML_CANN)
- set(CANN_INCLUDE_DIRS
- ${CANN_INSTALL_DIR}/include
- ${CANN_INSTALL_DIR}/include/aclnn
- ${CANN_INSTALL_DIR}/acllib/include
- )
-
- add_subdirectory(ggml-cann/kernels)
- list(APPEND CANN_LIBRARIES
- ascendcl
- nnopbase
- opapi
- acl_op_compiler
- ascendc_kernels
- )
-
- set(GGML_HEADERS_CANN "../include/ggml-cann.h")
- file(GLOB GGML_SOURCES_CANN "ggml-cann/*.cpp")
- list(APPEND GGML_SOURCES_CANN "ggml-cann.cpp")
-
- message(STATUS "CANN: CANN_INCLUDE_DIRS = ${CANN_INCLUDE_DIRS}")
- message(STATUS "CANN: CANN_LIBRARIES = ${CANN_LIBRARIES}")
-
- list(APPEND GGML_EXTRA_LIBS_PRIVATE ${CANN_LIBRARIES} )
- list(APPEND GGML_EXTRA_INCLUDES ${CANN_INCLUDE_DIRS})
- list(APPEND GGML_EXTRA_LIBDIRS ${CANN_INSTALL_DIR}/lib64)
-
- list(APPEND GGML_CDEF_PUBLIC GGML_USE_CANN)
- endif()
- else()
- set(GGML_CANN OFF)
- message(WARNING "CANN: Can't find CANN_INSTALL_DIR, do you forget to source set_var.sh. Turning off GGML_CANN")
- endif()
-
- if(NOT GGML_CANN)
- message(WARNING "CANN: GGML_CANN is turned OFF, see above for details.")
- endif()
-endif()
-
function(get_flags CCID CCVER)
set(C_FLAGS "")
set(CXX_FLAGS "")
elseif (CCID STREQUAL "GNU")
set(C_FLAGS -Wdouble-promotion)
set(CXX_FLAGS -Wno-array-bounds)
-
- if (NOT GGML_MUSA)
- if (CCVER VERSION_GREATER_EQUAL 7.1.0)
- list(APPEND CXX_FLAGS -Wno-format-truncation)
- endif()
- endif()
if (CCVER VERSION_GREATER_EQUAL 8.1.0)
list(APPEND CXX_FLAGS -Wextra-semi)
endif()
endif()
endif()
-set(CUDA_CXX_FLAGS "")
-
-if (GGML_CUDA)
- set(CUDA_FLAGS -use_fast_math)
-
- if (GGML_FATAL_WARNINGS)
- list(APPEND CUDA_FLAGS -Werror all-warnings)
- endif()
-
- if (GGML_ALL_WARNINGS AND NOT MSVC)
- set(NVCC_CMD ${CMAKE_CUDA_COMPILER} .c)
- if (NOT CMAKE_CUDA_HOST_COMPILER STREQUAL "")
- list(APPEND NVCC_CMD -ccbin ${CMAKE_CUDA_HOST_COMPILER})
- endif()
-
- execute_process(
- COMMAND ${NVCC_CMD} -Xcompiler --version
- OUTPUT_VARIABLE CUDA_CCFULLVER
- ERROR_QUIET
- )
-
- if (NOT CUDA_CCFULLVER MATCHES clang)
- set(CUDA_CCID "GNU")
- execute_process(
- COMMAND ${NVCC_CMD} -Xcompiler "-dumpfullversion -dumpversion"
- OUTPUT_VARIABLE CUDA_CCVER
- ERROR_QUIET
- )
- else()
- if (CUDA_CCFULLVER MATCHES Apple)
- set(CUDA_CCID "AppleClang")
- else()
- set(CUDA_CCID "Clang")
- endif()
- string(REGEX REPLACE "^.* version ([0-9.]*).*$" "\\1" CUDA_CCVER ${CUDA_CCFULLVER})
- endif()
-
- message("-- CUDA host compiler is ${CUDA_CCID} ${CUDA_CCVER}")
-
- get_flags(${CUDA_CCID} ${CUDA_CCVER})
- list(APPEND CUDA_CXX_FLAGS ${CXX_FLAGS} ${GF_CXX_FLAGS}) # This is passed to -Xcompiler later
- endif()
-
- if (NOT MSVC)
- list(APPEND CUDA_CXX_FLAGS -Wno-pedantic)
- endif()
-endif()
-
if (GGML_LTO)
include(CheckIPOSupported)
check_ipo_supported(RESULT result OUTPUT output)
endif()
endif()
-set(ARCH_FLAGS "")
-
-if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR
- CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR
- (NOT CMAKE_OSX_ARCHITECTURES AND
- NOT CMAKE_GENERATOR_PLATFORM_LWR AND
- CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$"))
-
- message(STATUS "ARM detected")
-
- if (MSVC)
- add_compile_definitions(__aarch64__) # MSVC defines _M_ARM64 instead
- add_compile_definitions(__ARM_NEON)
- add_compile_definitions(__ARM_FEATURE_FMA)
-
- set(CMAKE_REQUIRED_FLAGS_PREV ${CMAKE_REQUIRED_FLAGS})
- string(JOIN " " CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} "/arch:armv8.2")
-
- check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_DOTPROD)
- if (GGML_COMPILER_SUPPORT_DOTPROD)
- add_compile_definitions(__ARM_FEATURE_DOTPROD)
- endif ()
-
- check_cxx_source_compiles("#include <arm_neon.h>\nint main() { int8x16_t _a, _b; int32x4_t _s = vmlaq_f32(_s, _a, _b); return 0; }" GGML_COMPILER_SUPPORT_MATMUL_INT8)
-
- if (GGML_COMPILER_SUPPORT_MATMUL_INT8)
- add_compile_definitions(__ARM_FEATURE_MATMUL_INT8)
- endif ()
-
- check_cxx_source_compiles("#include <arm_neon.h>\nint main() { float16_t _a; float16x8_t _s = vdupq_n_f16(_a); return 0; }" GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
- if (GGML_COMPILER_SUPPORT_FP16_VECTOR_ARITHMETIC)
- add_compile_definitions(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
- endif ()
-
- set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_PREV})
- else()
- check_cxx_compiler_flag(-mfp16-format=ieee COMPILER_SUPPORTS_FP16_FORMAT_I3E)
- if (NOT "${COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "")
- list(APPEND ARCH_FLAGS -mfp16-format=ieee)
- endif()
- if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv6")
- # Raspberry Pi 1, Zero
- list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access)
- endif()
- if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv7")
- if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
- # Android armeabi-v7a
- list(APPEND ARCH_FLAGS -mfpu=neon-vfpv4 -mno-unaligned-access -funsafe-math-optimizations)
- else()
- # Raspberry Pi 2
- list(APPEND ARCH_FLAGS -mfpu=neon-fp-armv8 -mno-unaligned-access -funsafe-math-optimizations)
- endif()
- endif()
- if (${CMAKE_SYSTEM_PROCESSOR} MATCHES "armv8")
- # Android arm64-v8a
- # Raspberry Pi 3, 4, Zero 2 (32-bit)
- list(APPEND ARCH_FLAGS -mno-unaligned-access)
- endif()
- if (GGML_SVE)
- list(APPEND ARCH_FLAGS -march=armv8.6-a+sve)
- endif()
- endif()
-elseif (CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64" OR CMAKE_GENERATOR_PLATFORM_LWR MATCHES "^(x86_64|i686|amd64|x64|win32)$" OR
- (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND
- CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|i686|AMD64)$"))
- message(STATUS "x86 detected")
- if (MSVC)
- # instruction set detection for MSVC only
- if (GGML_NATIVE)
- # TODO: improve, should not reference files from the parent folder
- include(../cmake/FindSIMD.cmake)
- endif ()
- if (GGML_AVX512)
- list(APPEND ARCH_FLAGS /arch:AVX512)
- # MSVC has no compile-time flags enabling specific
- # AVX512 extensions, neither it defines the
- # macros corresponding to the extensions.
- # Do it manually.
- if (GGML_AVX512_VBMI)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
- endif()
- if (GGML_AVX512_VNNI)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
- endif()
- if (GGML_AVX512_BF16)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512BF16__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512BF16__>)
- endif()
- if (GGML_AMX_TILE)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_TILE__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_TILE__>)
- endif()
- if (GGML_AMX_INT8)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_INT8__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_INT8__>)
- endif()
- if (GGML_AMX_BF16)
- add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_BF16__>)
- add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_BF16__>)
- endif()
- elseif (GGML_AVX2)
- list(APPEND ARCH_FLAGS /arch:AVX2)
- elseif (GGML_AVX)
- list(APPEND ARCH_FLAGS /arch:AVX)
- endif()
- else()
- if (GGML_NATIVE)
- list(APPEND ARCH_FLAGS -march=native)
- endif()
- if (GGML_F16C)
- list(APPEND ARCH_FLAGS -mf16c)
- endif()
- if (GGML_FMA)
- list(APPEND ARCH_FLAGS -mfma)
- endif()
- if (GGML_AVX)
- list(APPEND ARCH_FLAGS -mavx)
- endif()
- if (GGML_AVX2)
- list(APPEND ARCH_FLAGS -mavx2)
- endif()
- if (GGML_AVX512)
- list(APPEND ARCH_FLAGS -mavx512f)
- list(APPEND ARCH_FLAGS -mavx512dq)
- list(APPEND ARCH_FLAGS -mavx512bw)
- endif()
- if (GGML_AVX512_VBMI)
- list(APPEND ARCH_FLAGS -mavx512vbmi)
- endif()
- if (GGML_AVX512_VNNI)
- list(APPEND ARCH_FLAGS -mavx512vnni)
- endif()
- if (GGML_AVX512_BF16)
- list(APPEND ARCH_FLAGS -mavx512bf16)
- endif()
- if (GGML_AMX_TILE)
- list(APPEND ARCH_FLAGS -mamx-tile)
- endif()
- if (GGML_AMX_INT8)
- list(APPEND ARCH_FLAGS -mamx-int8)
- endif()
- if (GGML_AMX_BF16)
- list(APPEND ARCH_FLAGS -mamx-bf16)
- endif()
- endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64")
- message(STATUS "PowerPC detected")
- execute_process(COMMAND bash -c "grep POWER10 /proc/cpuinfo | head -n 1" OUTPUT_VARIABLE POWER10_M)
- string(FIND "${POWER10_M}" "POWER10" substring_index)
- if (NOT DEFINED substring_index OR "${substring_index}" STREQUAL "")
- set(substring_index -1)
- endif()
-
- if (${substring_index} GREATER_EQUAL 0)
- list(APPEND ARCH_FLAGS -mcpu=power10)
- elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le")
- list(APPEND ARCH_FLAGS -mcpu=powerpc64le)
- else()
- list(APPEND ARCH_FLAGS -mcpu=native -mtune=native)
- #TODO: Add targets for Power8/Power9 (Altivec/VSX) and Power10(MMA) and query for big endian systems (ppc64/le/be)
- endif()
-elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "loongarch64")
- message(STATUS "loongarch64 detected")
-
- list(APPEND ARCH_FLAGS -march=loongarch64)
- if (GGML_LASX)
- list(APPEND ARCH_FLAGS -mlasx)
- endif()
- if (GGML_LSX)
- list(APPEND ARCH_FLAGS -mlsx)
- endif()
-else()
- message(STATUS "Unknown architecture")
-endif()
-
-add_compile_options("$<$<COMPILE_LANGUAGE:CXX>:${ARCH_FLAGS}>")
-add_compile_options("$<$<COMPILE_LANGUAGE:C>:${ARCH_FLAGS}>")
-
-if (GGML_CUDA)
- list(APPEND CUDA_CXX_FLAGS ${ARCH_FLAGS})
- list(JOIN CUDA_CXX_FLAGS " " CUDA_CXX_FLAGS_JOINED) # pass host compiler flags as a single argument
-
- if (NOT CUDA_CXX_FLAGS_JOINED STREQUAL "")
- list(APPEND CUDA_FLAGS -Xcompiler ${CUDA_CXX_FLAGS_JOINED})
- endif()
-
- add_compile_options("$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_FLAGS}>")
-endif()
-
if (MINGW)
# Target Windows 8 for PrefetchVirtualMemory
add_compile_definitions(_WIN32_WINNT=${GGML_WIN_VER})
# CLOCK_MONOTONIC came in POSIX.1-2001 / SUSv3 as optional
# posix_memalign came in POSIX.1-2001 / SUSv3
# M_PI is an XSI extension since POSIX.1-2001 / SUSv3, came in XPG1 (1985)
-add_compile_definitions(_XOPEN_SOURCE=600)
# Somehow in OpenBSD whenever POSIX conformance is specified
# some string functions rely on locale_t availability,
# which was introduced in POSIX.1-2008, forcing us to go higher
if (CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
- remove_definitions(-D_XOPEN_SOURCE=600)
add_compile_definitions(_XOPEN_SOURCE=700)
+else()
+ add_compile_definitions(_XOPEN_SOURCE=600)
endif()
# Data types, macros and functions related to controlling CPU affinity and
endif()
endif()
-#
-# libraries
-#
-
# ggml
-add_library(ggml
+add_library(ggml-base
../include/ggml.h
- ../include/ggml-cpu.h
../include/ggml-alloc.h
../include/ggml-backend.h
../include/ggml-cpp.h
ggml.c
- ggml-cpu.c
ggml-alloc.c
ggml-backend.cpp
+ ggml-threading.cpp
+ ggml-threading.h
ggml-quants.c
ggml-quants.h
- ${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA}
- ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL}
- ${GGML_SOURCES_RPC} ${GGML_HEADERS_RPC}
- ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
- ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL}
- ${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE}
- ${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN}
- ${GGML_SOURCES_ROCM} ${GGML_HEADERS_ROCM}
- ${GGML_SOURCES_BLAS} ${GGML_HEADERS_BLAS}
- ${GGML_SOURCES_LLAMAFILE} ${GGML_HEADERS_LLAMAFILE}
- ${GGML_SOURCES_AMX} ${GGML_HEADERS_AMX}
- ${GGML_SOURCES_CANN} ${GGML_HEADERS_CANN}
- ggml-aarch64.c ggml-aarch64.h
- )
+ ggml-aarch64.c
+ ggml-aarch64.h)
-if (EMSCRIPTEN)
- set_target_properties(ggml PROPERTIES COMPILE_FLAGS "-msimd128")
-endif()
+target_include_directories(ggml-base PRIVATE .)
-target_compile_definitions(ggml PUBLIC ${GGML_CDEF_PUBLIC})
-target_include_directories(ggml PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
-target_include_directories(ggml PRIVATE . ${GGML_EXTRA_INCLUDES})
-target_link_directories (ggml PRIVATE ${GGML_EXTRA_LIBDIRS})
-target_compile_features (ggml PRIVATE c_std_11) # don't bump
+add_library(ggml
+ ggml-backend-reg.cpp)
+
+target_link_libraries(ggml PUBLIC ggml-base)
+
+function(ggml_add_backend backend)
+ string(TOUPPER "GGML_${backend}" backend_id)
+ if (${backend_id})
+ string(TOLOWER "ggml-${backend}" backend_target)
+ add_subdirectory(${backend_target})
+ # check again in case the backend disabled itself
+ # note that this should NOT be the normal behavior, in case of errors the backend should fail the build
+ # however, currently it is necessary for AMX, since it is enabled by default on llama.cpp
+ if (${backend_id})
+ message(STATUS "Including ${backend} backend")
+ if (${BUILD_SHARED_LIBS})
+ target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
+ target_compile_definitions(${backend_target} PUBLIC GGML_BACKEND_SHARED)
+ install(TARGETS ${backend_target} LIBRARY)
+ endif()
+ target_link_libraries(ggml PUBLIC ${backend_target})
+ string(TOUPPER "GGML_USE_${backend}" backend_use)
+ target_compile_definitions(ggml PUBLIC ${backend_use})
+ endif()
+ endif()
+endfunction()
-list(APPEND GGML_EXTRA_LIBS_PRIVATE Threads::Threads)
+ggml_add_backend(CPU)
+ggml_add_backend(AMX)
+ggml_add_backend(BLAS)
+ggml_add_backend(CANN)
+ggml_add_backend(CUDA)
+ggml_add_backend(HIP)
+ggml_add_backend(Kompute)
+ggml_add_backend(METAL)
+ggml_add_backend(RPC)
+ggml_add_backend(SYCL)
+ggml_add_backend(Vulkan)
+ggml_add_backend(MUSA)
+
+foreach (target ggml-base ggml)
+ target_include_directories(${target} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
+ target_compile_features (${target} PRIVATE c_std_11) # don't bump
+endforeach()
+
+target_link_libraries(ggml-base PRIVATE Threads::Threads)
find_library(MATH_LIBRARY m)
if (MATH_LIBRARY)
if (NOT WIN32 OR NOT DEFINED ENV{ONEAPI_ROOT})
- list(APPEND GGML_EXTRA_LIBS_PRIVATE m)
+ target_link_libraries(ggml-base PRIVATE m)
endif()
endif()
if (CMAKE_SYSTEM_NAME MATCHES "Android")
- list(APPEND GGML_EXTRA_LIBS_PRIVATE dl) # Must be linked explicitly
+ target_link_libraries(ggml-base PRIVATE dl)
endif()
-list(REMOVE_DUPLICATES GGML_EXTRA_LIBS_PRIVATE)
-list(REMOVE_DUPLICATES GGML_EXTRA_LIBS_PUBLIC)
-target_link_libraries(ggml PRIVATE ${GGML_EXTRA_LIBS_PRIVATE} PUBLIC ${GGML_EXTRA_LIBS_PUBLIC})
-
if (BUILD_SHARED_LIBS)
- set_target_properties(ggml PROPERTIES POSITION_INDEPENDENT_CODE ON)
- target_compile_definitions(ggml PRIVATE GGML_SHARED GGML_BUILD)
+ foreach (target ggml-base ggml)
+ set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+ target_compile_definitions(${target} PRIVATE GGML_BUILD)
+ target_compile_definitions(${target} PUBLIC GGML_SHARED)
+ endforeach()
endif()
overview / pkg / ggml / sources / whisper.cpp / commitdiff