#include "ggml-backend-impl.h"
+#include "ggml-feats.h"
-#if defined(__aarch64__)
-
-#if defined(__linux__)
-#include <sys/auxv.h>
-#elif defined(__APPLE__)
-#include <sys/sysctl.h>
-#endif
-
-#if !defined(HWCAP_FPHP)
-#define HWCAP_FPHP (1 << 9)
-#endif
-
-#if !defined(HWCAP_ASIMDHP)
-#define HWCAP_ASIMDHP (1 << 10)
-#endif
-
-#if !defined(HWCAP_ASIMDDP)
-#define HWCAP_ASIMDDP (1 << 20)
-#endif
-
-#if !defined(HWCAP_SVE)
-#define HWCAP_SVE (1 << 22)
-#endif
-
-#if !defined(HWCAP2_SVE2)
-#define HWCAP2_SVE2 (1 << 1)
-#endif
-
-#if !defined(HWCAP2_I8MM)
-#define HWCAP2_I8MM (1 << 13)
-#endif
-
-#if !defined(HWCAP2_SME)
-#define HWCAP2_SME (1 << 23)
-#endif
-
-struct aarch64_features {
- // has_neon not needed, aarch64 has NEON guaranteed
- bool has_dotprod = false;
- bool has_fp16 = false;
- bool has_sve = false;
- bool has_sve2 = false;
- bool has_i8mm = false;
- bool has_sme = false;
- bool has_sme2 = false;
-
- aarch64_features() {
-#if defined(__linux__)
- uint32_t hwcap = getauxval(AT_HWCAP);
- uint32_t hwcap2 = getauxval(AT_HWCAP2);
-
- has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
- has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);
- has_sve = !!(hwcap & HWCAP_SVE);
- has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
- has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
- has_sme = !!(hwcap2 & HWCAP2_SME);
-#elif defined(__APPLE__)
- int oldp = 0;
- size_t size = sizeof(oldp);
-
- if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, NULL, 0) == 0) {
- has_dotprod = static_cast<bool>(oldp);
- }
-
- if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, NULL, 0) == 0) {
- has_i8mm = static_cast<bool>(oldp);
- }
-
- if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, NULL, 0) == 0) {
- has_sme = static_cast<bool>(oldp);
- }
-
- if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, NULL, 0) == 0) {
- has_sme2 = static_cast<bool>(oldp);
- }
-
- // Apple apparently does not implement SVE yet
-#endif
- }
-};
+#if defined(__aarch64__) || defined(_M_ARM64)
static int ggml_backend_cpu_aarch64_score() {
int score = 1;
- aarch64_features af;
+ ggml_feats_arch64_runtime_t af = ggml_get_aarch64_runtime_features();
#ifdef GGML_USE_DOTPROD
if (!af.has_dotprod) { return 0; }
GGML_BACKEND_DL_SCORE_IMPL(ggml_backend_cpu_aarch64_score)
-# endif // defined(__aarch64__)
+# endif // defined(__aarch64__) || defined(_M_ARM64)
// SPDX-License-Identifier: MIT
//
#include <arm_neon.h>
-#include <assert.h>
-#include <stdio.h>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
#include <atomic>
#include <cfloat>
+#include <cctype>
#include <algorithm>
#include <cmath>
#include <stdexcept>
#include <cstddef>
#include <cstdint>
#include <fstream>
-#include <set>
+#include <map>
#include <iostream>
#include <climits>
+#include <charconv>
+#include <system_error>
#if defined(__linux__)
#include <asm/hwcap.h>
+#include <dirent.h>
#include <sys/auxv.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
-#ifndef HWCAP2_SME2
-#define HWCAP2_SME2 (1UL << 37)
-#endif
#elif defined(__APPLE__)
-#include <string_view>
#include <sys/sysctl.h>
#include <sys/types.h>
-#elif defined(_WIN32)
-#include <windows.h>
-#include <excpt.h>
#endif
#include "kleidiai.h"
#include "ggml-cpu.h"
#include "ggml-cpu-impl.h"
#include "ggml-impl.h"
+#include "ggml-feats.h"
#include "ggml-backend-impl.h"
#include "ggml-threading.h"
#include "traits.h"
ggml_kleidiai_kernels * kernels_q4;
ggml_kleidiai_kernels * kernels_q8;
ggml_kleidiai_kernels * kernels_f32;
- int sme_thread_cap; // <= 0 means “SME disabled/unknown”;
- int thread_hint; // <= 0 means “no hint”
+ int sme_thread_cap; // <= 0 means "SME disabled/unknown"
+ int thread_hint; // <= 0 means "no hint"
int chunk_multiplier;
} static ctx = { CPU_FEATURE_NONE, nullptr, nullptr, nullptr, 0, -1, 4 };
}
}
+#if defined(__linux__) && defined(__aarch64__)
+static bool parse_cpu_dir_name(const char* name, size_t* cpu) {
+ if (strncmp(name, "cpu", 3) != 0 ||
+ name[3] < '0' || name[3] > '9') {
+ return false;
+ }
+
+ const char* first = name + 3;
+ const char* last = name + strlen(name);
+
+ size_t value = 0;
+ const auto [end, ec] = std::from_chars(first, last, value, 10);
+
+ if (ec != std::errc{} || end != last) {
+ return false;
+ }
+
+ *cpu = value;
+ return true;
+}
+
+static std::vector<size_t> detect_cpu_ids() {
+ std::vector<size_t> cpus;
+
+ DIR * dir = opendir("/sys/devices/system/cpu");
+ if (dir == nullptr) {
+ return cpus;
+ }
+
+ while (dirent * entry = readdir(dir)) {
+ size_t cpu = 0;
+ if (parse_cpu_dir_name(entry->d_name, &cpu)) {
+ cpus.push_back(cpu);
+ }
+ }
+ closedir(dir);
+
+ std::sort(cpus.begin(), cpus.end());
+ cpus.erase(std::unique(cpus.begin(), cpus.end()), cpus.end());
+ return cpus;
+}
+#endif
+
+#if defined(__APPLE__) && defined(__aarch64__)
+static bool apple_sme_counted_perf_level(std::string name) {
+ for (std::string::size_type i = 0; i < name.size(); ++i) {
+ name[i] = (char) std::tolower((unsigned char) name[i]);
+ }
+
+ // Conservative ceiling: only count perf-level names observed to provide full SME throughput.
+ // Future names should be calibrated here before they raise the automatic SME thread cap.
+ return name.find("super") != std::string::npos ||
+ name.find("performance") != std::string::npos;
+}
+#endif
+
+static void add_smcus_from_smidr(uint64_t smidr, size_t & num_private, std::map<uint32_t, size_t> & shared_counts) {
+ // Arm ARM: SMIDR_EL1. SH==0 is implementation-defined; keep the existing
+ // conservative policy and only treat zero affinity as private.
+ const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
+ const uint32_t nsmc = (uint32_t)((smidr >> 56) & 0xF);
+ const size_t shared_count = nsmc == 0xF ? 1 : (size_t)nsmc + 1;
+ const uint32_t affinity = (uint32_t)(smidr & 0xFFFu);
+ const uint32_t affinity2 = (uint32_t)((smidr >> 32) & 0xFFFFFu);
+ const uint32_t id = (affinity2 << 12) | affinity;
+
+ if (nsmc == 0xF) {
+ GGML_LOG_WARN("kleidiai: NSMC detected as 0xF indicating reseved value, setting min safe shared SMCU count to 1");
+ }
+
+ switch (sh) {
+ case 2: // private SMCU
+ ++num_private;
+ break;
+ case 3: // shared SMCU
+ if (shared_counts[id] < shared_count) {
+ shared_counts[id] = shared_count;
+ }
+ break;
+ case 0:
+ if (id == 0) {
+ ++num_private;
+ } else if (shared_counts[id] < shared_count) {
+ shared_counts[id] = shared_count;
+ }
+ break;
+ default:
+ break;
+ }
+}
+
static size_t detect_num_smcus() {
- if (!ggml_cpu_has_sme()) {
+ auto runtime_feat = ggml_get_aarch64_runtime_features();
+ if (!runtime_feat.has_sme) {
return 0;
}
#if defined(__linux__) && defined(__aarch64__)
// Linux/aarch64: Best-effort count of Streaming Mode Compute Units (SMCUs) via SMIDR_EL1 sysfs.
size_t num_private = 0;
- std::set<uint32_t> shared_ids;
+ std::map<uint32_t, size_t> shared_counts;
- for (size_t cpu = 0;; ++cpu) {
+ const std::vector<size_t> cpus = detect_cpu_ids();
+ for (const size_t cpu : cpus) {
const std::string path =
"/sys/devices/system/cpu/cpu" + std::to_string(cpu) +
"/regs/identification/smidr_el1";
std::ifstream file(path);
if (!file.is_open()) {
- break;
+ continue;
}
uint64_t smidr = 0;
continue;
}
- // Arm ARM: SMIDR_EL1
- const uint32_t sh = (uint32_t)((smidr >> 13) & 0x3);
- // Build an "affinity-like" identifier for shared SMCUs.
- // Keep the original packing logic, but isolate it here.
- const uint32_t id = (uint32_t)((smidr & 0xFFFu) | ((smidr >> 20) & 0xFFFFF000u));
-
- switch (sh) {
- case 0b10: // private SMCU
- ++num_private;
- break;
- case 0b11: // shared SMCU
- shared_ids.emplace(id);
- break;
- case 0b00:
- // Ambiguous / implementation-defined. Be conservative:
- // treat id==0 as private, otherwise as shared.
- if (id == 0) ++num_private;
- else shared_ids.emplace(id);
- break;
- default:
- break;
- }
+ add_smcus_from_smidr(smidr, num_private, shared_counts);
}
- return num_private + shared_ids.size();
+ size_t total = num_private;
+ for (const auto & entry : shared_counts) {
+ total += entry.second;
+ }
+ return total;
#elif defined(__APPLE__) && defined(__aarch64__)
- // table for known M4 variants. Users can override via GGML_KLEIDIAI_SME=<n>.
- char chip_name[256] = {};
- size_t size = sizeof(chip_name);
-
- if (sysctlbyname("machdep.cpu.brand_string", chip_name, &size, nullptr, 0) == 0) {
- const std::string brand(chip_name);
-
- struct ModelSMCU { const char *match; size_t smcus; };
- static const ModelSMCU table[] = {
- { "M4 Ultra", 2 },
- { "M4 Max", 2 },
- { "M4 Pro", 2 },
- { "M4", 1 },
- };
+ int perf_levels = 0;
+ size_t size = sizeof(perf_levels);
+ if (sysctlbyname("hw.nperflevels", &perf_levels, &size, nullptr, 0) != 0 ||
+ size != sizeof(perf_levels) || perf_levels <= 0) {
+ return 0;
+ }
- for (const auto &e : table) {
- if (brand.find(e.match) != std::string::npos) {
- return e.smcus;
- }
+ size_t units = 0;
+ for (int i = 0; i < perf_levels; ++i) {
+ char key[64] = {};
+ int physical_cpus = 0;
+ int cpus_per_l2 = 0;
+
+ snprintf(key, sizeof(key), "hw.perflevel%d.physicalcpu", i);
+ size = sizeof(physical_cpus);
+ if (sysctlbyname(key, &physical_cpus, &size, nullptr, 0) != 0 ||
+ size != sizeof(physical_cpus) || physical_cpus <= 0) {
+ continue;
+ }
+
+ snprintf(key, sizeof(key), "hw.perflevel%d.cpusperl2", i);
+ size = sizeof(cpus_per_l2);
+ if (sysctlbyname(key, &cpus_per_l2, &size, nullptr, 0) != 0 ||
+ size != sizeof(cpus_per_l2) || cpus_per_l2 <= 0) {
+ continue;
+ }
+
+ snprintf(key, sizeof(key), "hw.perflevel%d.name", i);
+ size = 0;
+ if (sysctlbyname(key, nullptr, &size, nullptr, 0) != 0 || size == 0) {
+ continue;
+ }
+
+ std::string name(size, '\0');
+ if (sysctlbyname(key, &name[0], &size, nullptr, 0) != 0) {
+ continue;
+ }
+ name.resize(size);
+ while (!name.empty() && name.back() == '\0') {
+ name.pop_back();
+ }
+
+ if (apple_sme_counted_perf_level(name)) {
+ units += (size_t) ((physical_cpus + cpus_per_l2 - 1) / cpus_per_l2);
}
}
+
+ return units;
+
+#elif defined(_WIN32) && (defined(_M_ARM64) || defined(__aarch64__))
+ // No verified Windows arm64 SMCU detection path yet. Return unknown and use
+ // GGML_KLEIDIAI_SME=N as a diagnostics/debug override for SME thread cap
+ // calibration until a detection mechanism is verified on real hardware.
return 0;
#else
if (!initialized) {
initialized = true;
+ // Optional diagnostics/debug overrides; production defaults come from runtime detection.
const char *env_sme = getenv("GGML_KLEIDIAI_SME");
const char *env_threads = getenv("GGML_TOTAL_THREADS");
const char *env_chunk_mult = getenv("GGML_KLEIDIAI_CHUNK_MULTIPLIER");
+ auto runtime_feat = ggml_get_aarch64_runtime_features();
+
size_t detected_smcus = 0;
- ctx.features = (ggml_cpu_has_dotprod() ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
- (ggml_cpu_has_matmul_int8() ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |
- ((ggml_cpu_has_sve() && ggml_cpu_get_sve_cnt() == QK8_0) ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
+ ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
+ (runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |
+ (runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
if (env_threads) {
bool ok = false;
}
}
- // SME policy:
- // - env unset => auto-detect SMCUs; enable SME only if detected > 0.
- // - env=0 => force off.
- // - env>0 => force N cores, if the binary was built with SME.
int sme_cores = 0;
bool sme_env_ok = false;
bool sme_env_set = (env_sme != nullptr);
+ const bool has_supported_sme_family = runtime_feat.has_sme;
+ bool sme_cap_detected = false;
+
+ if (has_supported_sme_family) {
+ detected_smcus = detect_num_smcus();
+ sme_cap_detected = detected_smcus > 0;
+ // Some platforms expose SME without exposing a calibrated SMCU count.
+ // Use one SME thread as the conservative default; add platform SMCU detection to raise it.
+ sme_cores = sme_cap_detected ? (int)detected_smcus : 1;
+
+ if (!sme_env_set && !sme_cap_detected) {
+ GGML_LOG_INFO("kleidiai: SME detected; SMCU count unavailable, using conservative SME thread cap=1\n");
+ }
+ }
+
+ // Runtime-detect SME support and available SMCUs first. The detected SMCU
+ // count is used as the SME thread cap, and GGML_KLEIDIAI_SME can debug-override that:
+ // - unset: use runtime detection.
+ // - 0: disable SME-family kernels.
+ // - N > 0: use N as the SME thread cap, if an SME-family kernel is selectable.
if (sme_env_set) {
bool ok = false;
int v = parse_uint_env(env_sme, "GGML_KLEIDIAI_SME", &ok);
sme_env_ok = ok;
- if (!ok) {
- GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; falling back to runtime SME-core detection\n");
- detected_smcus = detect_num_smcus();
- sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
- } else if (v == 0) {
- sme_cores = 0;
- } else if (!ggml_cpu_has_sme()) {
- GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but the binary was not built with SME; disabling SME\n", v);
- sme_cores = 0;
+ if (ok) {
+ if (has_supported_sme_family) {
+ sme_cores = v;
+ } else {
+ if (v > 0) {
+ GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME=%d but SME is not supported on this CPU; disabling SME-family kernels\n", v);
+ }
+ sme_cores = 0;
+ }
} else {
- sme_cores = v;
+ GGML_LOG_WARN("kleidiai: GGML_KLEIDIAI_SME set but parsing failed; using automatic SME thread cap\n");
}
- } else {
- detected_smcus = detect_num_smcus();
- sme_cores = detected_smcus > 0 ? (int)detected_smcus : 0;
}
- if (!sme_env_set && ggml_cpu_has_sme() && sme_cores == 0) {
- GGML_LOG_WARN("kleidiai: runtime SME-core detection returned 0; falling back to NEON\n");
- }
-
- if (sme_cores > 0) {
+ if (sme_cores > 0 && has_supported_sme_family) {
ctx.features |= CPU_FEATURE_SME;
-#if defined(__aarch64__) && defined(__linux__)
- // ARM guarantees SME2 implies SME, so only check SME2 when SME is enabled.
- if (getauxval(AT_HWCAP2) & HWCAP2_SME2) {
+ if (runtime_feat.has_sme2) {
ctx.features |= CPU_FEATURE_SME2;
}
-#elif defined(__aarch64__) && defined(__APPLE__)
- int feat_sme2 = 0;
- size_t size = sizeof(feat_sme2);
- if (sysctlbyname("hw.optional.arm.FEAT_SME2", &feat_sme2, &size, NULL, 0) == 0 && feat_sme2) {
- ctx.features |= CPU_FEATURE_SME2;
- }
-#endif
}
// Kernel selection
GGML_LOG_INFO("kleidiai: primary f32 kernel feature %s\n", cpu_feature_to_string(ctx.kernels_f32->required_cpu));
}
- ctx.sme_thread_cap = (ctx.features & CPU_FEATURE_SME) ? sme_cores : 0;
+ const bool has_selected_sme_family_kernel =
+ (ctx.kernels_q4 && is_sme_family(ctx.kernels_q4->required_cpu)) ||
+ (ctx.kernels_q8 && is_sme_family(ctx.kernels_q8->required_cpu)) ||
+ (ctx.kernels_f32 && is_sme_family(ctx.kernels_f32->required_cpu));
+ ctx.sme_thread_cap = has_selected_sme_family_kernel ? sme_cores : 0;
- if (ctx.features & CPU_FEATURE_SME) {
- const bool has_sme2 = (ctx.features & CPU_FEATURE_SME2) != CPU_FEATURE_NONE;
+ if (has_selected_sme_family_kernel) {
if (sme_env_set && sme_env_ok && sme_cores > 0) {
- GGML_LOG_INFO("kleidiai: SME%s enabled (GGML_KLEIDIAI_SME=%d override)\n",
- has_sme2 ? "2" : "", sme_cores);
+ GGML_LOG_INFO("kleidiai: SME enabled (GGML_KLEIDIAI_SME=%d debug override)\n", sme_cores);
+ } else if (sme_cap_detected) {
+ GGML_LOG_INFO("kleidiai: SME enabled (runtime-detected SME thread cap=%d)\n", sme_cores);
} else {
- GGML_LOG_INFO("kleidiai: SME%s enabled (runtime-detected SME cores=%d)\n",
- has_sme2 ? "2" : "", sme_cores);
+ GGML_LOG_INFO("kleidiai: SME enabled (runtime SME detected, conservative thread cap=%d)\n", sme_cores);
}
} else {
GGML_LOG_INFO("kleidiai: SME disabled\n");
}
if (is_sme_family(primary->required_cpu)) {
- const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~CPU_FEATURE_SME & ~CPU_FEATURE_SME2);
+ const cpu_feature fallback_mask = static_cast<cpu_feature>(features & ~(CPU_FEATURE_SME | CPU_FEATURE_SME2));
if (fallback_mask != CPU_FEATURE_NONE) {
ggml_kleidiai_kernels * fallback = select_fallback(fallback_mask);
if (fallback && fallback != primary &&
const int ith_total = params->ith;
int sme_slot = -1;
+ int non_sme_slot = -1;
for (int i = 0; i < runtime_count; ++i) {
if (is_sme_family(runtime[i].kernels->required_cpu)) {
sme_slot = i;
break;
}
}
- int non_sme_slot = -1;
+
for (int i = 0; i < runtime_count; ++i) {
if (!is_sme_family(runtime[i].kernels->required_cpu)) {
non_sme_slot = i;
--- /dev/null
+#pragma once
+
+#if defined(__aarch64__) || defined(_M_ARM64)
+
+#if defined(__linux__)
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+
+#if !defined(HWCAP2_SVE2)
+#define HWCAP2_SVE2 (1ULL << 1)
+#endif
+
+#if !defined(HWCAP_FPHP)
+#define HWCAP_FPHP (1 << 9)
+#endif
+
+#if !defined(HWCAP_ASIMDHP)
+#define HWCAP_ASIMDHP (1 << 10)
+#endif
+
+#if !defined(HWCAP2_I8MM)
+#define HWCAP2_I8MM (1ULL << 13)
+#endif
+
+#if !defined(HWCAP_ASIMDDP)
+#define HWCAP_ASIMDDP (1 << 20)
+#endif
+
+#if !defined(HWCAP_SVE)
+#define HWCAP_SVE (1 << 22)
+#endif
+
+#if !defined(HWCAP2_SME)
+#define HWCAP2_SME (1ULL << 23)
+#endif
+
+#if !defined(HWCAP2_SME2)
+#define HWCAP2_SME2 (1ULL << 37)
+#endif
+
+#if !defined(PR_SVE_GET_VL)
+#define PR_SVE_GET_VL 51
+#endif
+
+#if !defined(PR_SVE_VL_LEN_MASK)
+#define PR_SVE_VL_LEN_MASK 0xffff
+#endif
+
+#elif defined(__APPLE__)
+#include <sys/sysctl.h>
+#elif defined(_WIN32)
+#include <windows.h>
+
+#if !defined(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43
+#endif
+
+#if !defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46
+#endif
+
+#if !defined(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
+#endif
+
+#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
+#endif
+
+#if !defined(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE 67
+#endif
+
+#if !defined(PF_ARM_SME_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SME_INSTRUCTIONS_AVAILABLE 70
+#endif
+
+#if !defined(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
+#endif
+
+#endif
+
+typedef struct ggml_feats_arch64_runtime {
+ bool has_dotprod;
+ bool has_fp16;
+ bool has_sve;
+ bool has_sve2;
+ bool has_i8mm;
+ bool has_sme;
+ bool has_sme2;
+ int sve_cnt;
+} ggml_feats_arch64_runtime_t;
+
+static inline ggml_feats_arch64_runtime_t ggml_get_aarch64_runtime_features(void) {
+ ggml_feats_arch64_runtime_t runtime_feat = {};
+
+#if defined(__linux__)
+ const unsigned long hwcap = getauxval(AT_HWCAP);
+ const unsigned long hwcap2 = getauxval(AT_HWCAP2);
+
+ runtime_feat.has_dotprod = !!(hwcap & HWCAP_ASIMDDP);
+ runtime_feat.has_fp16 = !!(hwcap & HWCAP_FPHP) && !!(hwcap & HWCAP_ASIMDHP);;
+ runtime_feat.has_sve = !!(hwcap & HWCAP_SVE);
+ runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
+ runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
+ runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME);
+ runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2);
+
+ if (runtime_feat.has_sve) {
+ const int vl = prctl(PR_SVE_GET_VL);
+ if (vl >= 0) {
+ runtime_feat.sve_cnt = vl & PR_SVE_VL_LEN_MASK;
+ }
+ }
+#elif defined(__APPLE__)
+ int oldp = 0;
+ size_t size = sizeof(oldp);
+
+ if (sysctlbyname("hw.optional.arm.FEAT_DotProd", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_dotprod = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_FP16", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_fp16 = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_SVE", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_sve = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_SVE2", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_sve2 = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_I8MM", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_i8mm = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_SME", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_sme = static_cast<bool>(oldp);
+ }
+
+ if (sysctlbyname("hw.optional.arm.FEAT_SME2", &oldp, &size, nullptr, 0) == 0) {
+ runtime_feat.has_sme2 = static_cast<bool>(oldp);
+ }
+
+ // Apple does not support userspace non-streaming SVE; keep SVE vector length unknown.
+ runtime_feat.sve_cnt = 0;
+#elif defined (_WIN32)
+ runtime_feat.has_dotprod = IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_fp16 = IsProcessorFeaturePresent(PF_ARM_V82_FP16_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;
+
+ // Windows exposes SVE feature presence, but not the runtime SVE vector length here.
+ runtime_feat.sve_cnt = 0;
+#endif
+
+ return runtime_feat;
+}
+
+#endif // defined(__aarch64__) || defined(_M_ARM64)
overview / pkg / ggml / sources / llama.cpp / commitdiff