endif()
option(BUILD_SHARED_LIBS "ggml: build shared libraries" ${BUILD_SHARED_LIBS_DEFAULT})
+option(GGML_BACKEND_DL "ggml: build backends as dynamic libraries (requires BUILD_SHARED_LIBS)" OFF)
#
# option list
typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
// Get additional buffer types provided by the device (returns a NULL-terminated array)
typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
+ // Set the abort callback for the backend
+ typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
+ // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
+ struct ggml_backend_feature {
+ const char * name;
+ const char * value;
+ };
+ typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
//
// Backend registry
// = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
GGML_API ggml_backend_t ggml_backend_init_best(void);
+ // Load a backend from a dynamic library and register it
+ GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
+ // Unload a backend if loaded dynamically and unregister it
+ GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);
+ // Load all known backends from dynamic libraries
+ GGML_API void ggml_backend_load_all(void);
+
//
// Backend scheduler
//
extern "C" {
#endif
- // Scheduling priorities
- enum ggml_sched_priority {
- GGML_SCHED_PRIO_NORMAL,
- GGML_SCHED_PRIO_MEDIUM,
- GGML_SCHED_PRIO_HIGH,
- GGML_SCHED_PRIO_REALTIME
- };
-
- // Threadpool params
- // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
- struct ggml_threadpool_params {
- bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
- int n_threads; // number of threads
- enum ggml_sched_priority prio; // thread priority
- uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
- bool strict_cpu; // strict cpu placement
- bool paused; // start in paused state
- };
-
- struct ggml_threadpool; // forward declaration, see ggml.c
-
- typedef struct ggml_threadpool * ggml_threadpool_t;
-
// the compute plan that needs to be prepared for ggml_graph_compute()
// since https://github.com/ggerganov/ggml/issues/287
struct ggml_cplan {
GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3);
GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value);
- GGML_BACKEND_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
- GGML_BACKEND_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);
- GGML_BACKEND_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
- GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params);
- GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool);
- GGML_BACKEND_API int ggml_threadpool_get_n_threads(struct ggml_threadpool * threadpool);
- GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool);
- GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool);
+ GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params);
+ GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool);
+ GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool);
+ GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool);
+ GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool);
// ggml_graph_plan() has to be called before ggml_graph_compute()
// when plan.work_size > 0, caller must allocate memory for plan.work_data
GGML_BACKEND_API int ggml_cpu_has_sse3 (void);
GGML_BACKEND_API int ggml_cpu_has_ssse3 (void);
GGML_BACKEND_API int ggml_cpu_has_avx (void);
+ GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void);
GGML_BACKEND_API int ggml_cpu_has_avx2 (void);
GGML_BACKEND_API int ggml_cpu_has_f16c (void);
GGML_BACKEND_API int ggml_cpu_has_fma (void);
- GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void);
GGML_BACKEND_API int ggml_cpu_has_avx512 (void);
GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void);
GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void);
GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type);
+ // ggml threadpool
+ // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend
+ // the goal should be to create an API that other backends can use move everything to the ggml base
+
+ // scheduling priorities
+ enum ggml_sched_priority {
+ GGML_SCHED_PRIO_NORMAL,
+ GGML_SCHED_PRIO_MEDIUM,
+ GGML_SCHED_PRIO_HIGH,
+ GGML_SCHED_PRIO_REALTIME
+ };
+
+ // threadpool params
+ // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults
+ struct ggml_threadpool_params {
+ bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings)
+ int n_threads; // number of threads
+ enum ggml_sched_priority prio; // thread priority
+ uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling)
+ bool strict_cpu; // strict cpu placement
+ bool paused; // start in paused state
+ };
+
+ struct ggml_threadpool; // forward declaration, see ggml.c
+
+ typedef struct ggml_threadpool * ggml_threadpool_t;
+
+ GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads);
+ GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads);
+ GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1);
+
#ifdef __cplusplus
}
#endif
# ggml
+if (GGML_BACKEND_DL AND NOT BUILD_SHARED_LIBS)
+ message(FATAL_ERROR "GGML_BACKEND_DL requires BUILD_SHARED_LIBS")
+endif()
+
add_library(ggml-base
../include/ggml.h
../include/ggml-alloc.h
target_link_libraries(ggml PUBLIC ggml-base)
+if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+ target_link_libraries(ggml PRIVATE dl)
+endif()
+
+function(ggml_add_backend_library backend)
+ if (GGML_BACKEND_DL)
+ add_library(${backend} MODULE ${ARGN})
+ # write the shared library to the output directory
+ set_target_properties(${backend} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY})
+ target_compile_definitions(${backend} PRIVATE GGML_BACKEND_DL)
+ else()
+ add_library(${backend} ${ARGN})
+ target_link_libraries(ggml PUBLIC ${backend})
+ install(TARGETS ${backend} LIBRARY)
+ endif()
+
+ target_link_libraries(${backend} PRIVATE ggml-base)
+ target_include_directories(${backend} PRIVATE ..)
+
+ if (${BUILD_SHARED_LIBS})
+ target_compile_definitions(${backend} PRIVATE GGML_BACKEND_BUILD)
+ target_compile_definitions(${backend} PUBLIC GGML_BACKEND_SHARED)
+ endif()
+endfunction()
+
function(ggml_add_backend backend)
string(TOUPPER "GGML_${backend}" backend_id)
if (${backend_id})
# however, currently it is necessary for AMX, since it is enabled by default on llama.cpp
if (${backend_id})
message(STATUS "Including ${backend} backend")
- if (${BUILD_SHARED_LIBS})
- target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
- target_compile_definitions(${backend_target} PUBLIC GGML_BACKEND_SHARED)
+ if (NOT GGML_BACKEND_DL)
+ string(TOUPPER "GGML_USE_${backend}" backend_use)
+ target_compile_definitions(ggml PUBLIC ${backend_use})
endif()
- install(TARGETS ${backend_target} LIBRARY)
- target_link_libraries(ggml PUBLIC ${backend_target})
- string(TOUPPER "GGML_USE_${backend}" backend_use)
- target_compile_definitions(ggml PUBLIC ${backend_use})
endif()
endif()
endfunction()
ggml_add_backend(HIP)
ggml_add_backend(Kompute)
ggml_add_backend(METAL)
+ggml_add_backend(MUSA)
ggml_add_backend(RPC)
ggml_add_backend(SYCL)
ggml_add_backend(Vulkan)
-ggml_add_backend(MUSA)
foreach (target ggml-base ggml)
target_include_directories(${target} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
extern "C" {
#endif
+ #define GGML_BACKEND_API_VERSION 1
+
//
// Backend buffer type
//
enum ggml_backend_buffer_usage usage;
};
- ggml_backend_buffer_t ggml_backend_buffer_init(
+ GGML_API ggml_backend_buffer_t ggml_backend_buffer_init(
ggml_backend_buffer_type_t buft,
struct ggml_backend_buffer_i iface,
void * context,
size_t size);
// do not use directly, use ggml_backend_tensor_copy instead
- bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
+ GGML_API bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
// multi-buffer
// buffer that contains a collection of buffers
- ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
- bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
- void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
+ GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
+ GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
+ GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
//
// Backend (stream)
};
struct ggml_backend_reg {
- // int api_version; // TODO: for dynamic loading
+ int api_version; // initialize to GGML_BACKEND_API_VERSION
struct ggml_backend_reg_i iface;
void * context;
};
-
// Internal backend registry API
- void ggml_backend_register(ggml_backend_reg_t reg);
- void ggml_backend_device_register(ggml_backend_dev_t device);
- // TODO: backends can be loaded as a dynamic library, in which case it needs to export this function
- // typedef ggml_backend_register_t * (*ggml_backend_init)(void);
+ GGML_API void ggml_backend_register(ggml_backend_reg_t reg);
+ GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
+
+ // Add backend dynamic loading support to the backend
+ typedef ggml_backend_reg_t (*ggml_backend_init_t)(void);
+
+ #ifdef GGML_BACKEND_DL
+ #ifdef __cplusplus
+ # define GGML_BACKEND_DL_IMPL(reg_fn) \
+ extern "C" { \
+ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \
+ } \
+ ggml_backend_reg_t ggml_backend_init(void) { \
+ return reg_fn(); \
+ }
+ #else
+ # define GGML_BACKEND_DL_IMPL(reg_fn) \
+ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_init(void); \
+ ggml_backend_reg_t ggml_backend_init(void) { \
+ return reg_fn(); \
+ }
+ #endif
+ #else
+ # define GGML_BACKEND_DL_IMPL(reg_fn)
+ #endif
#ifdef __cplusplus
}
#include "ggml-backend-impl.h"
#include "ggml-backend.h"
-#include "ggml-cpu.h"
#include "ggml-impl.h"
+#include <algorithm>
#include <cstring>
+#include <string>
#include <vector>
+#ifdef _WIN32
+# define WIN32_LEAN_AND_MEAN
+# ifndef NOMINMAX
+# define NOMINMAX
+# endif
+# include <windows.h>
+#elif defined(__APPLE__)
+# include <mach-o/dyld.h>
+# include <dlfcn.h>
+#else
+# include <dlfcn.h>
+# include <unistd.h>
+#endif
+
// Backend registry
+#ifdef GGML_USE_CPU
+#include "ggml-cpu.h"
+#endif
#ifdef GGML_USE_CUDA
#include "ggml-cuda.h"
#include "ggml-kompute.h"
#endif
+struct ggml_backend_reg_entry {
+ ggml_backend_reg_t reg;
+ void * handle;
+};
+
struct ggml_backend_registry {
- std::vector<ggml_backend_reg_t> backends;
+ std::vector<ggml_backend_reg_entry> backends;
std::vector<ggml_backend_dev_t> devices;
ggml_backend_registry() {
#ifdef GGML_USE_KOMPUTE
register_backend(ggml_backend_kompute_reg());
#endif
-
+#ifdef GGML_USE_CPU
register_backend(ggml_backend_cpu_reg());
+#endif
}
- void register_backend(ggml_backend_reg_t reg) {
+ ~ggml_backend_registry() {
+ while (!backends.empty()) {
+ // use silent since the log system may have been destroyed at this point
+ unload_backend(backends.back().reg, true);
+ }
+ }
+
+ void register_backend(ggml_backend_reg_t reg, void * handle = nullptr) {
if (!reg) {
return;
}
GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",
__func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));
#endif
- backends.push_back(reg);
+ backends.push_back({ reg, handle });
for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {
register_device(ggml_backend_reg_dev_get(reg, i));
}
#endif
devices.push_back(device);
}
+
+ ggml_backend_reg_t load_backend(const char * path, bool silent) {
+#ifdef _WIN32
+ // suppress error dialogs for missing DLLs
+ DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
+ SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);
+
+ HMODULE handle = LoadLibraryA(path);
+
+ if (!handle) {
+ if (!silent) {
+ GGML_LOG_ERROR("%s: failed to load %s: %lu\n", __func__, path, GetLastError());
+ }
+ SetErrorMode(old_mode);
+ return nullptr;
+ }
+
+ ggml_backend_init_t backend_init = (ggml_backend_init_t) GetProcAddress(handle, "ggml_backend_init");
+
+ SetErrorMode(old_mode);
+
+ if (!backend_init) {
+ if (!silent) {
+ GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %lu\n", __func__, path, GetLastError());
+ }
+ FreeLibrary(handle);
+ return nullptr;
+ }
+#else
+ void * handle = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+
+ if (!handle) {
+ if (!silent) {
+ GGML_LOG_ERROR("%s: failed to load %s: %s\n", __func__, path, dlerror());
+ }
+ return nullptr;
+ }
+
+ auto * backend_init = (ggml_backend_init_t) dlsym(handle, "ggml_backend_init");
+
+ if (!backend_init) {
+ if (!silent) {
+ GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s: %s\n", __func__, path, dlerror());
+ }
+ dlclose(handle);
+ return nullptr;
+ }
+#endif
+ ggml_backend_reg_t reg = backend_init();
+
+ if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {
+ if (!silent) {
+ if (!reg) {
+ GGML_LOG_ERROR("%s: failed to initialize backend from %s: ggml_backend_init returned NULL\n", __func__, path);
+ } else {
+ GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",
+ __func__, path, reg->api_version, GGML_BACKEND_API_VERSION);
+ }
+ }
+#ifdef _WIN32
+ FreeLibrary(handle);
+#else
+ dlclose(handle);
+#endif
+ return nullptr;
+ }
+
+ GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), path);
+ register_backend(reg, handle);
+ return reg;
+ }
+
+ void unload_backend(ggml_backend_reg_t reg, bool silent) {
+ auto it = std::find_if(backends.begin(), backends.end(),
+ [reg](ggml_backend_reg_entry entry) { return entry.reg == reg; });
+
+ if (it == backends.end()) {
+ if (!silent) {
+ GGML_LOG_ERROR("%s: backend not found\n", __func__);
+ }
+ return;
+ }
+
+ if (!silent) {
+ GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));
+ }
+
+ // remove devices
+ devices.erase(
+ std::remove_if(devices.begin(), devices.end(),
+ [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),
+ devices.end());
+
+ // unload library
+ if (it->handle) {
+#ifdef _WIN32
+ FreeLibrary((HMODULE) it->handle);
+#else
+ dlclose(it->handle);
+#endif
+ }
+
+ // remove backend
+ backends.erase(it);
+ }
};
static ggml_backend_registry & get_reg() {
ggml_backend_reg_t ggml_backend_reg_get(size_t index) {
GGML_ASSERT(index < ggml_backend_reg_count());
- return get_reg().backends[index];
+ return get_reg().backends[index].reg;
}
ggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {
return reg;
}
}
- return NULL;
+ return nullptr;
}
// Device enumeration
return dev;
}
}
- return NULL;
+ return nullptr;
}
ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {
return dev;
}
}
- return NULL;
+ return nullptr;
}
// Convenience functions
ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {
ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);
if (!dev) {
- return NULL;
+ return nullptr;
}
return ggml_backend_dev_init(dev, params);
}
ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {
ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);
if (!dev) {
- return NULL;
+ return nullptr;
}
return ggml_backend_dev_init(dev, params);
}
dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
}
if (!dev) {
- return NULL;
+ return nullptr;
}
- return ggml_backend_dev_init(dev, NULL);
+ return ggml_backend_dev_init(dev, nullptr);
+}
+
+// Dynamic loading
+ggml_backend_reg_t ggml_backend_load(const char * path) {
+ return get_reg().load_backend(path, false);
+}
+
+void ggml_backend_unload(ggml_backend_reg_t reg) {
+ get_reg().unload_backend(reg, true);
+}
+
+void ggml_backend_load_all() {
+ std::vector<std::string> search_prefix;
+
+ // add the executable directory to the search path
+ // FIXME: this is convenient for development, but it should probably be disabled in production
+
+#if defined(__APPLE__)
+ // get executable path
+ std::vector<char> path;
+ uint32_t size;
+ while (true) {
+ size = path.size();
+ if (_NSGetExecutablePath(path.data(), &size) == 0) {
+ break;
+ }
+ path.resize(size);
+ }
+ std::string base_path(path.data(), size);
+ // remove executable name
+ auto last_slash = base_path.find_last_of('/');
+ if (last_slash != std::string::npos) {
+ base_path = base_path.substr(0, last_slash);
+ }
+ search_prefix.push_back(base_path + "/");
+#elif defined(__linux__)
+ std::string base_path = ".";
+ std::vector<char> path(1024);
+ while (true) {
+ // get executable path
+ ssize_t len = readlink("/proc/self/exe", path.data(), path.size());
+ if (len == -1) {
+ break;
+ }
+ if (len < (ssize_t) path.size()) {
+ base_path = std::string(path.data(), len);
+ // remove executable name
+ auto last_slash = base_path.find_last_of('/');
+ if (last_slash != std::string::npos) {
+ base_path = base_path.substr(0, last_slash);
+ }
+ break;
+ }
+ path.resize(path.size() * 2);
+ }
+
+ search_prefix.push_back(base_path + "/");
+#endif
+
+ auto & reg = get_reg();
+
+ auto try_load = [&](const std::string & name) {
+ std::string os_name;
+#ifdef _WIN32
+ os_name = "ggml-" + name + ".dll";
+#else
+ os_name = "libggml-" + name + ".so";
+#endif
+ if (reg.load_backend(os_name.c_str(), true)) {
+ return;
+ }
+ for (const auto & prefix : search_prefix) {
+ if (reg.load_backend((prefix + os_name).c_str(), true)) {
+ return;
+ }
+ }
+ };
+
+ try_load("amx");
+ try_load("blas");
+ try_load("cann");
+ try_load("cuda");
+ try_load("hip");
+ try_load("kompute");
+ try_load("metal");
+ try_load("rpc");
+ try_load("sycl");
+ try_load("vulkan");
+ try_load("musa");
+ try_load("cpu");
}
if (BLAS_FOUND)
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
- add_library(ggml-blas
- ggml-blas.cpp
- )
-
- target_link_libraries(ggml-blas PRIVATE ggml-base)
- target_include_directories(ggml-blas PRIVATE . ..)
+ ggml_add_backend_library(ggml-blas
+ ggml-blas.cpp
+ )
if (${GGML_BLAS_VENDOR} MATCHES "Apple")
add_compile_definitions(ACCELERATE_NEW_LAPACK)
ggml_backend_reg_t ggml_backend_blas_reg(void) {
static struct ggml_backend_reg ggml_backend_blas_reg = {
- /* .iface = */ ggml_backend_blas_reg_i,
- /* .context = */ NULL,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_blas_reg_i,
+ /* .context = */ NULL,
};
return &ggml_backend_blas_reg;
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_blas_reg)
file(GLOB GGML_SOURCES_CANN "*.cpp")
- add_library(ggml-cann ${GGML_SOURCES_CANN})
- target_link_libraries(ggml-cann PRIVATE ggml-base ${CANN_LIBRARIES})
- target_include_directories(ggml-cann PRIVATE . .. ${CANN_INCLUDE_DIRS})
+ ggml_add_backend_library(ggml-cann ${GGML_SOURCES_CANN})
+ target_link_libraries(ggml-cann PRIVATE ${CANN_LIBRARIES})
+ target_include_directories(ggml-cann PRIVATE ${CANN_INCLUDE_DIRS})
target_link_directories(ggml-cann PRIVATE ${CANN_INSTALL_DIR}/lib64)
target_compile_definitions(ggml-cann PRIVATE "-D${SOC_TYPE_COMPILE_OPTION}")
dev_ctx->name = GGML_CANN_NAME + std::to_string(i);
ggml_cann_set_device(i);
ggml_backend_dev_t dev = new ggml_backend_device {
- /* .interface = */ ggml_backend_cann_device_interface,
- /* .reg = */ ®,
- /* .context = */ dev_ctx
+ /* .iface = */ ggml_backend_cann_device_interface,
+ /* .reg = */ ®,
+ /* .context = */ dev_ctx
};
ctx->devices.push_back(dev);
}
reg = ggml_backend_reg {
- /* .interface = */ ggml_backend_cann_reg_interface,
- /* .context = */ ctx
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_cann_reg_interface,
+ /* .context = */ ctx
};
}
ggml_cann_set_device(device);
ACL_CHECK(aclrtGetMemInfo(ACL_HBM_MEM, free, total));
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_cann_reg)
-add_library(ggml-cpu
- ggml-cpu.c
- ggml-cpu.cpp
- ggml-cpu-aarch64.c
- ggml-cpu-aarch64.h
- ggml-cpu-quants.c
- ggml-cpu-quants.h
- )
+ggml_add_backend_library(ggml-cpu
+ ggml-cpu.c
+ ggml-cpu.cpp
+ ggml-cpu-aarch64.c
+ ggml-cpu-aarch64.h
+ ggml-cpu-quants.c
+ ggml-cpu-quants.h
+ )
-target_link_libraries(ggml-cpu PRIVATE ggml-base)
-target_include_directories(ggml-cpu PRIVATE . ..)
+target_include_directories(ggml-cpu PRIVATE .)
if (APPLE AND GGML_ACCELERATE)
find_library(ACCELERATE_FRAMEWORK Accelerate)
#endif // GGML_USE_OPENMP
-void ggml_threadpool_params_init(struct ggml_threadpool_params * p, int n_threads) {
- p->n_threads = n_threads;
- p->prio = 0; // default priority (usually means normal or inherited)
- p->poll = 50; // hybrid-polling enabled
- p->strict_cpu = false; // no strict placement (all threads share same cpumask)
- p->paused = false; // threads are ready to go
- memset(p->cpumask, 0, GGML_MAX_N_THREADS); // all-zero means use the default affinity (usually inherited)
-}
-
-struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads) {
- struct ggml_threadpool_params p;
- ggml_threadpool_params_init(&p, n_threads);
- return p;
-}
-
-bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1) {
- if (p0->n_threads != p1->n_threads ) return false;
- if (p0->prio != p1->prio ) return false;
- if (p0->poll != p1->poll ) return false;
- if (p0->strict_cpu != p1->strict_cpu ) return false;
- return memcmp(p0->cpumask, p1->cpumask, GGML_MAX_N_THREADS) == 0;
-}
-
static struct ggml_threadpool * ggml_threadpool_new_impl(
struct ggml_threadpool_params * tpp,
struct ggml_cgraph * cgraph,
return &ggml_backend_cpu_device;
}
-struct ggml_backend_feature {
- const char * name;
- const char * value;
-};
-
-// Not used yet
// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,
-// and additionally to allow other backends to expose their own list of features that applications can query using the same API.
+// and additionally to allow other backends to expose their own list of features that applications can query using the same API
static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t reg) {
static std::vector<ggml_backend_feature> features = []() {
+ ggml_cpu_init();
+
std::vector<ggml_backend_feature> features;
if (ggml_cpu_has_sse3()) {
features.push_back({ "SSE3", "1" });
if (ggml_cpu_has_avx()) {
features.push_back({ "AVX", "1" });
}
+ if (ggml_cpu_has_avx_vnni()) {
+ features.push_back({ "AVX_VNNI", "1" });
+ }
if (ggml_cpu_has_avx2()) {
features.push_back({ "AVX2", "1" });
}
if (ggml_cpu_has_fma()) {
features.push_back({ "FMA", "1" });
}
- if (ggml_cpu_has_avx_vnni()) {
- features.push_back({ "AVX_VNNI", "1" });
- }
if (ggml_cpu_has_avx512()) {
features.push_back({ "AVX512", "1" });
}
if (ggml_cpu_has_llamafile()) {
features.push_back({ "LLAMAFILE", "1" });
}
+ // TODO: rename this
+ #ifdef GGML_USE_CPU_AARCH64
+ features.push_back({ "AARCH64_REPACK", "1" });
+ #endif
features.push_back({ nullptr, nullptr });
if (strcmp(name, "ggml_backend_dev_get_extra_bufts") == 0) {
return (void *)ggml_backend_cpu_get_extra_bufts;
}
+ if (strcmp(name, "ggml_backend_get_features") == 0) {
+ return (void *)ggml_backend_cpu_get_features;
+ }
+ if (strcmp(name, "ggml_backend_set_abort_callback") == 0) {
+ return (void *)ggml_backend_cpu_set_abort_callback;
+ }
+ if (strcmp(name, "ggml_backend_cpu_numa_init") == 0) {
+ return (void *)ggml_numa_init;
+ }
+ if (strcmp(name, "ggml_backend_cpu_is_numa") == 0) {
+ return (void *)ggml_is_numa;
+ }
+
+ // threadpool - TODO: move to ggml-base
+ if (strcmp(name, "ggml_threadpool_new") == 0) {
+ return (void *)ggml_threadpool_new;
+ }
+ if (strcmp(name, "ggml_threadpool_free") == 0) {
+ return (void *)ggml_threadpool_free;
+ }
+ if (strcmp(name, "ggml_backend_cpu_set_threadpool") == 0) {
+ return (void *)ggml_backend_cpu_set_threadpool;
+ }
return NULL;
ggml_cpu_init();
static struct ggml_backend_reg ggml_backend_cpu_reg = {
- /* .iface = */ ggml_backend_cpu_reg_i,
- /* .context = */ NULL,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_cpu_reg_i,
+ /* .context = */ NULL,
};
return &ggml_backend_cpu_reg;
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_cpu_reg)
return ctx->devices[index];
}
+static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t reg) {
+ static std::vector<ggml_backend_feature> features = []() {
+ std::vector<ggml_backend_feature> features;
+ #define _STRINGIFY(...) #__VA_ARGS__
+ #define STRINGIFY(...) _STRINGIFY(__VA_ARGS__)
+
+ #ifdef __CUDA_ARCH_LIST__
+ features.push_back({ "ARCHS", STRINGIFY(__CUDA_ARCH_LIST__) });
+ #endif
+
+ #ifdef GGML_CUDA_FORCE_MMQ
+ features.push_back({ "FORCE_MMQ", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_FORCE_CUBLAS
+ features.push_back({ "FORCE_CUBLAS", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_NO_VMM
+ features.push_back({ "NO_VMM", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_NO_PEER_COPY
+ features.push_back({ "NO_PEER_COPY", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_F16
+ features.push_back({ "F16", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_USE_GRAPHS
+ features.push_back({ "USE_GRAPHS", "1" });
+ #endif
+
+ #ifdef GGML_CUDA_PEER_MAX_BATCH_SIZE
+ features.push_back({ "PEER_MAX_BATCH_SIZE", STRINGIFY(GGML_CUDA_PEER_MAX_BATCH_SIZE) });
+ #endif
+
+ #ifdef GGML_CUDA_FA_ALL_QUANTS
+ features.push_back({ "FA_ALL_QUANTS", "1" });
+ #endif
+
+ #undef _STRINGIFY
+ #undef STRINGIFY
+
+ features.push_back({ nullptr, nullptr });
+
+ return features;
+ }();
+
+ return features.data();
+
+ GGML_UNUSED(reg);
+}
+
static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
GGML_UNUSED(reg);
if (strcmp(name, "ggml_backend_split_buffer_type") == 0) {
if (strcmp(name, "ggml_backend_unregister_host_buffer") == 0) {
return (void *)ggml_backend_cuda_unregister_host_buffer;
}
+ if (strcmp(name, "ggml_backend_get_features") == 0) {
+ return (void *)ggml_backend_cuda_get_features;
+ }
return nullptr;
}
dev_ctx->description = prop.name;
ggml_backend_dev_t dev = new ggml_backend_device {
- /* .interface = */ ggml_backend_cuda_device_interface,
- /* .reg = */ ®,
- /* .context = */ dev_ctx
+ /* .iface = */ ggml_backend_cuda_device_interface,
+ /* .reg = */ ®,
+ /* .context = */ dev_ctx
};
ctx->devices.push_back(dev);
}
reg = ggml_backend_reg {
- /* .interface = */ ggml_backend_cuda_reg_interface,
- /* .context = */ ctx
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_cuda_reg_interface,
+ /* .context = */ ctx
};
}
return cuda_backend;
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_cuda_reg)
list(APPEND GGML_SOURCES_CUDA ${SRCS})
endif()
- add_library(ggml-cuda
- ${GGML_HEADERS_CUDA}
- ${GGML_SOURCES_CUDA}
- )
-
- target_link_libraries(ggml-cuda PRIVATE ggml-base)
- target_include_directories(ggml-cuda PRIVATE . ..)
+ ggml_add_backend_library(ggml-cuda
+ ${GGML_HEADERS_CUDA}
+ ${GGML_SOURCES_CUDA}
+ )
add_compile_definitions(GGML_CUDA_PEER_MAX_BATCH_SIZE=${GGML_CUDA_PEER_MAX_BATCH_SIZE})
list(APPEND GGML_SOURCES_ROCM ${SRCS})
endif()
-add_library(ggml-hip
- ${GGML_HEADERS_ROCM}
- ${GGML_SOURCES_ROCM})
-
-target_link_libraries(ggml-hip PRIVATE ggml-base)
-target_include_directories(ggml-hip PRIVATE . ..)
+ggml_add_backend_library(ggml-hip
+ ${GGML_HEADERS_ROCM}
+ ${GGML_SOURCES_ROCM}
+ )
# TODO: do not use CUDA definitions for HIP
target_compile_definitions(ggml PUBLIC GGML_USE_CUDA)
message(FATAL_ERROR "glslc not found")
endif()
-add_library(ggml-kompute
- ggml-kompute.cpp
- ../../include/ggml-kompute.h
- )
+ggml_add_backend_library(ggml-kompute
+ ggml-kompute.cpp
+ ../../include/ggml-kompute.h
+ )
target_link_libraries(ggml-kompute PRIVATE ggml-base kompute)
-target_include_directories(ggml-kompute PRIVATE . .. ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(ggml-kompute PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
add_compile_definitions(VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1)
ggml_backend_reg_t ggml_backend_kompute_reg() {
static ggml_backend_reg reg = {
- /* .iface = */ ggml_backend_kompute_reg_i,
- /* .context = */ nullptr,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_kompute_reg_i,
+ /* .context = */ nullptr,
};
return ®
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_kompute_reg)
message(STATUS "Metal framework found")
-add_library(ggml-metal
- ggml-metal.m
- )
+ggml_add_backend_library(ggml-metal
+ ggml-metal.m
+ )
target_link_libraries(ggml-metal PRIVATE
- ggml-base
${FOUNDATION_LIBRARY}
${METAL_FRAMEWORK}
${METALKIT_FRAMEWORK}
)
-target_include_directories(ggml-metal PRIVATE . ..)
-
if (GGML_METAL_NDEBUG)
add_compile_definitions(GGML_METAL_NDEBUG)
endif()
GGML_UNUSED(index);
}
+static struct ggml_backend_feature g_ggml_backend_metal_features[] = {
+#if defined(GGML_METAL_EMBED_LIBRARY)
+ { "EMBED_LIBRARY", "1" },
+#endif
+#if defined(GGML_METAL_USE_BF16)
+ { "BF16", "1" },
+#endif
+ { nil, nil },
+};
+
+static struct ggml_backend_feature * ggml_backend_metal_get_features(ggml_backend_reg_t reg) {
+ return g_ggml_backend_metal_features;
+
+ GGML_UNUSED(reg);
+}
+
+static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {
+ if (strcmp(name, "ggml_backend_get_features") == 0) {
+ return (void *)ggml_backend_metal_get_features;
+ }
+
+ return NULL;
+
+ GGML_UNUSED(reg);
+}
static struct ggml_backend_reg_i ggml_backend_metal_reg_i = {
/* .get_name = */ ggml_backend_metal_reg_get_name,
/* .device_count = */ ggml_backend_metal_reg_device_count,
/* .device_get = */ ggml_backend_metal_reg_device_get,
- /* .get_proc_address = */ NULL,
+ /* .get_proc_address = */ ggml_backend_metal_get_proc_address,
};
ggml_backend_reg_t ggml_backend_metal_reg(void) {
// TODO: make this thread-safe somehow?
{
g_ggml_backend_metal_reg = (struct ggml_backend_reg) {
- /* .iface = */ ggml_backend_metal_reg_i,
- /* .context = */ NULL,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_metal_reg_i,
+ /* .context = */ NULL,
};
g_ggml_backend_metal_device = (struct ggml_backend_device) {
return &g_ggml_backend_metal_reg;
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_metal_reg)
set_property(SOURCE ${SOURCE} PROPERTY COMPILE_FLAGS "-x musa -mtgpu --cuda-gpu-arch=mp_21 --cuda-gpu-arch=mp_22")
endforeach()
- add_library(ggml-musa
- ${GGML_HEADERS_MUSA}
- ${GGML_SOURCES_MUSA})
-
- target_link_libraries(ggml-musa PRIVATE ggml-base)
- target_include_directories(ggml-musa PRIVATE . ..)
+ ggml_add_backend_library(ggml-musa
+ ${GGML_HEADERS_MUSA}
+ ${GGML_SOURCES_MUSA}
+ )
# TODO: do not use CUDA definitions for MUSA
target_compile_definitions(ggml PUBLIC GGML_USE_CUDA)
message(STATUS "Using RPC backend")
-add_library(ggml-rpc
- ggml-rpc.cpp)
-
-target_link_libraries(ggml-rpc PRIVATE ggml-base)
-target_include_directories(ggml-rpc PRIVATE . ..)
+ggml_add_backend_library(ggml-rpc
+ ggml-rpc.cpp
+ )
if (WIN32)
target_link_libraries(ggml-rpc PRIVATE ws2_32)
ggml_backend_reg_t ggml_backend_rpc_reg(void) {
static struct ggml_backend_reg ggml_backend_rpc_reg = {
- /* .iface = */ ggml_backend_rpc_reg_i,
- /* .context = */ NULL,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_rpc_reg_i,
+ /* .context = */ NULL,
};
return &ggml_backend_rpc_reg;
return dev;
}
+
+GGML_BACKEND_DL_IMPL(ggml_backend_rpc_reg)
message(STATUS "SYCL found")
#todo: AOT
-add_library(ggml-sycl
- ggml-sycl.cpp
- ../../include/ggml-sycl.h)
-
-target_link_libraries(ggml-sycl PRIVATE ggml-base)
-target_include_directories(ggml-sycl PRIVATE . ..)
+ggml_add_backend_library(ggml-sycl
+ ggml-sycl.cpp
+ ../../include/ggml-sycl.h
+ )
if (GGML_SYCL_F16)
if (GGML_SYCL_TARGET STREQUAL "AMD")
dev_ctx->description = prop.get_name();
ggml_backend_dev_t dev = new ggml_backend_device {
- /* .interface = */ ggml_backend_sycl_device_interface,
- /* .reg = */ ®,
- /* .context = */ dev_ctx
+ /* .iface = */ ggml_backend_sycl_device_interface,
+ /* .reg = */ ®,
+ /* .context = */ dev_ctx
};
ctx->devices.push_back(dev);
}
reg = ggml_backend_reg {
- /* .interface = */ ggml_backend_sycl_reg_interface,
- /* .context = */ ctx
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_sycl_reg_interface,
+ /* .context = */ ctx
};
}
return sycl_backend;
}
+GGML_BACKEND_DL_IMPL(ggml_backend_sycl_reg)
if (Vulkan_FOUND)
message(STATUS "Vulkan found")
- add_library(ggml-vulkan
- ggml-vulkan.cpp
- ../../include/ggml-vulkan.h
- )
+ ggml_add_backend_library(ggml-vulkan
+ ggml-vulkan.cpp
+ ../../include/ggml-vulkan.h
+ )
- target_link_libraries(ggml-vulkan PRIVATE ggml-base Vulkan::Vulkan)
- target_include_directories(ggml-vulkan PRIVATE . .. ${CMAKE_CURRENT_BINARY_DIR})
+ target_link_libraries(ggml-vulkan PRIVATE Vulkan::Vulkan)
+ target_include_directories(ggml-vulkan PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
# Workaround to the "can't dereference invalidated vector iterator" bug in clang-cl debug build
# Posssibly relevant: https://stackoverflow.com/questions/74748276/visual-studio-no-displays-the-correct-length-of-stdvector
ggml_backend_reg_t ggml_backend_vk_reg() {
static ggml_backend_reg reg = {
- /* .iface = */ ggml_backend_vk_reg_i,
- /* .context = */ nullptr,
+ /* .api_version = */ GGML_BACKEND_API_VERSION,
+ /* .iface = */ ggml_backend_vk_reg_i,
+ /* .context = */ nullptr,
};
return ®
VK_LOG_DEBUG("END ggml_vk_check_results_1(" << tensor->name << ")");
}
#endif
+
+GGML_BACKEND_DL_IMPL(ggml_backend_vk_reg)
g_logger_state.log_callback = log_callback ? log_callback : ggml_log_callback_default;
g_logger_state.log_callback_user_data = user_data;
}
+
+void ggml_threadpool_params_init(struct ggml_threadpool_params * p, int n_threads) {
+ p->n_threads = n_threads;
+ p->prio = 0; // default priority (usually means normal or inherited)
+ p->poll = 50; // hybrid-polling enabled
+ p->strict_cpu = false; // no strict placement (all threads share same cpumask)
+ p->paused = false; // threads are ready to go
+ memset(p->cpumask, 0, GGML_MAX_N_THREADS); // all-zero means use the default affinity (usually inherited)
+}
+
+struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads) {
+ struct ggml_threadpool_params p;
+ ggml_threadpool_params_init(&p, n_threads);
+ return p;
+}
+
+bool ggml_threadpool_params_match(const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1) {
+ if (p0->n_threads != p1->n_threads ) return false;
+ if (p0->prio != p1->prio ) return false;
+ if (p0->poll != p1->poll ) return false;
+ if (p0->strict_cpu != p1->strict_cpu ) return false;
+ return memcmp(p0->cpumask, p1->cpumask, GGML_MAX_N_THREADS) == 0;
+}
overview / pkg / ggml / sources / whisper.cpp / commitdiff