+++ /dev/null
-// SPDX-FileCopyrightText: Copyright 2024 Arm Ltd.
-#define GGML_COMMON_IMPL_C
-#include "ggml-common.h"
-
-#include "ggml-quants.h"
-#include "ggml-impl.h"
-
-#include <math.h>
-#include <string.h>
-#include <assert.h>
-#include <float.h>
-#include <stdlib.h> // for qsort
-#include <stdio.h> // for GGML_ASSERT
-
-#include "ggml-aarch64.h"
-
-#if defined(__GNUC__)
-#pragma GCC diagnostic ignored "-Woverlength-strings"
-#endif
-
-#define UNUSED GGML_UNUSED
-
-// Functions to create the interleaved data layout formats
-
-// interleave 4 block_q4_0s in blocks of blck_size_interleave
-// returns an interleaved block_q4_0x4
-// in the interleaved block_q4_0x4, place deltas for 4 block_q4_0 blocks
-// first, then interleave quants from 4 block_q4_0s in blocks of blck_size_interleave
-//
-// - in : an array of block_q4_0 pointers
-// - blck_size_interleave : the block_q4_0 quants bytes are interleaved in blocks of
-// blck_size_interleave bytes
-// - xor_mask : the mask to convert the nibbles in block_q4_0 quants bytes
-// from bias offset form to pure sign form (this saves subtract
-// operations durin unpacking)
-//
-static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave, unsigned int xor_mask) {
- block_q4_0x4 out;
-
- for (int i = 0; i < 4; i++) {
- out.d[i] = in[i].d;
- }
-
- for (int i = 0; i < QK4_0 * 2; i++) {
- int src_offset = (i / (4 * blck_size_interleave)) * blck_size_interleave;
- int src_id = (i % (4 * blck_size_interleave)) / blck_size_interleave;
- src_offset += (i % blck_size_interleave);
-
- out.qs[i] = in[src_id].qs[src_offset] ^ xor_mask;
- }
-
- return out;
-}
-
-// interleave 8 block_q4_0s in blocks of blck_size_interleave
-// returns an interleaved block_q4_0x8
-// in the interleaved block_q4_0x8, place deltas for 8 block_q4_0 blocks
-// first, then interleave quants from 8 block_q4_0s in blocks of blck_size_interleave
-static block_q4_0x8 make_block_q4_0x8(block_q4_0 * in, unsigned int blck_size_interleave, unsigned int xor_mask) {
- block_q4_0x8 out;
-
- for (int i = 0; i < 8; i++) {
- out.d[i] = in[i].d;
- }
-
- for (int i = 0; i < QK4_0 * 4; i++) {
- int src_offset = (i / (8 * blck_size_interleave)) * blck_size_interleave;
- int src_id = (i % (8 * blck_size_interleave)) / blck_size_interleave;
- src_offset += (i % blck_size_interleave);
-
- out.qs[i] = in[src_id].qs[src_offset] ^ xor_mask;
- }
-
- return out;
-}
-
-void quantize_q8_0_4x4(const float * restrict x, void * restrict vy, int64_t k) {
- assert(QK8_0 == 32);
- assert(k % QK8_0 == 0);
- const int nb = k / QK8_0;
-
- block_q8_0x4 * restrict y = (block_q8_0x4 *) vy;
-
-#if defined(__ARM_NEON)
- float32x4_t srcv[4][8];
- float id[4];
-
- for (int i = 0; i < nb; i++) {
- float32x4_t asrcv[8];
- float32x4_t amaxv[8];
-
- for (int row_iter = 0; row_iter < 4; row_iter++) {
- for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);
- for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);
-
- for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);
- for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);
- for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);
-
- const float amax = vmaxvq_f32(amaxv[0]);
-
- const float d = amax / ((1 << 7) - 1);
- id[row_iter] = d ? 1.0f / d : 0.0f;
-
- y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
- }
-
- for (int j = 0; j < 8; j++) {
- float32x4_t v = vmulq_n_f32(srcv[0][j], id[0]);
- int32x4_t vi = vcvtnq_s32_f32(v);
- y[i].qs[16 * j + 0] = vgetq_lane_s32(vi, 0);
- y[i].qs[16 * j + 1] = vgetq_lane_s32(vi, 1);
- y[i].qs[16 * j + 2] = vgetq_lane_s32(vi, 2);
- y[i].qs[16 * j + 3] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[1][j], id[1]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[16 * j + 4] = vgetq_lane_s32(vi, 0);
- y[i].qs[16 * j + 5] = vgetq_lane_s32(vi, 1);
- y[i].qs[16 * j + 6] = vgetq_lane_s32(vi, 2);
- y[i].qs[16 * j + 7] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[2][j], id[2]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[16 * j + 8] = vgetq_lane_s32(vi, 0);
- y[i].qs[16 * j + 9] = vgetq_lane_s32(vi, 1);
- y[i].qs[16 * j + 10] = vgetq_lane_s32(vi, 2);
- y[i].qs[16 * j + 11] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[3][j], id[3]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[16 * j + 12] = vgetq_lane_s32(vi, 0);
- y[i].qs[16 * j + 13] = vgetq_lane_s32(vi, 1);
- y[i].qs[16 * j + 14] = vgetq_lane_s32(vi, 2);
- y[i].qs[16 * j + 15] = vgetq_lane_s32(vi, 3);
- }
- }
-#else
- // scalar
- const int blck_size_interleave = 4;
- float srcv[4][QK8_0];
- float id[4];
-
- for (int i = 0; i < nb; i++) {
- for (int row_iter = 0; row_iter < 4; row_iter++) {
- float amax = 0.0f; // absolute max
-
- for (int j = 0; j < QK8_0; j++) {
- srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];
- amax = MAX(amax, fabsf(srcv[row_iter][j]));
- }
-
- const float d = amax / ((1 << 7) - 1);
- id[row_iter] = d ? 1.0f / d : 0.0f;
-
- y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
- }
-
- for (int j = 0; j < QK8_0 * 4; j++) {
- int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;
- int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;
- src_offset += (j % blck_size_interleave);
-
- float x0 = srcv[src_id][src_offset] * id[src_id];
- y[i].qs[j] = roundf(x0);
- }
- }
-#endif
-}
-
-void quantize_q8_0_4x8(const float * restrict x, void * restrict vy, int64_t k) {
- assert(QK8_0 == 32);
- assert(k % QK8_0 == 0);
- const int nb = k / QK8_0;
-
- block_q8_0x4 * restrict y = (block_q8_0x4 *) vy;
-
-#if defined(__ARM_NEON)
- float32x4_t srcv[4][8];
- float id[4];
-
- for (int i = 0; i < nb; i++) {
- float32x4_t asrcv[8];
- float32x4_t amaxv[8];
-
- for (int row_iter = 0; row_iter < 4; row_iter++) {
- for (int j = 0; j < 8; j++) srcv[row_iter][j] = vld1q_f32(x + row_iter * k + i * 32 + 4 * j);
- for (int j = 0; j < 8; j++) asrcv[j] = vabsq_f32(srcv[row_iter][j]);
-
- for (int j = 0; j < 4; j++) amaxv[2 * j] = vmaxq_f32(asrcv[2 * j], asrcv[2 * j + 1]);
- for (int j = 0; j < 2; j++) amaxv[4 * j] = vmaxq_f32(amaxv[4 * j], amaxv[4 * j + 2]);
- for (int j = 0; j < 1; j++) amaxv[8 * j] = vmaxq_f32(amaxv[8 * j], amaxv[8 * j + 4]);
-
- const float amax = vmaxvq_f32(amaxv[0]);
-
- const float d = amax / ((1 << 7) - 1);
- id[row_iter] = d ? 1.0f / d : 0.0f;
-
- y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
- }
-
- for (int j = 0; j < 4; j++) {
- float32x4_t v = vmulq_n_f32(srcv[0][2 * j], id[0]);
- int32x4_t vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 0] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 1] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 2] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 3] = vgetq_lane_s32(vi, 3);
- v = vmulq_n_f32(srcv[0][2 * j + 1], id[0]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 4] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 5] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 6] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 7] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[1][2 * j], id[1]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 8] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 9] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 10] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 11] = vgetq_lane_s32(vi, 3);
- v = vmulq_n_f32(srcv[1][2 * j + 1], id[1]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 12] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 13] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 14] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 15] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[2][2 * j], id[2]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 16] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 17] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 18] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 19] = vgetq_lane_s32(vi, 3);
- v = vmulq_n_f32(srcv[2][2 * j + 1], id[2]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 20] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 21] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 22] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 23] = vgetq_lane_s32(vi, 3);
-
- v = vmulq_n_f32(srcv[3][2 * j], id[3]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 24] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 25] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 26] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 27] = vgetq_lane_s32(vi, 3);
- v = vmulq_n_f32(srcv[3][2 * j + 1], id[3]);
- vi = vcvtnq_s32_f32(v);
- y[i].qs[32 * j + 28] = vgetq_lane_s32(vi, 0);
- y[i].qs[32 * j + 29] = vgetq_lane_s32(vi, 1);
- y[i].qs[32 * j + 30] = vgetq_lane_s32(vi, 2);
- y[i].qs[32 * j + 31] = vgetq_lane_s32(vi, 3);
- }
- }
-#else
- // scalar
- const int blck_size_interleave = 8;
- float srcv[4][QK8_0];
- float id[4];
-
- for (int i = 0; i < nb; i++) {
- for (int row_iter = 0; row_iter < 4; row_iter++) {
- float amax = 0.0f; // absolute max
-
- for (int j = 0; j < QK8_0; j++) {
- srcv[row_iter][j] = x[row_iter * k + i * QK8_0 + j];
- amax = MAX(amax, fabsf(srcv[row_iter][j]));
- }
-
- const float d = amax / ((1 << 7) - 1);
- id[row_iter] = d ? 1.0f / d : 0.0f;
-
- y[i].d[row_iter] = GGML_FP32_TO_FP16(d);
- }
-
- for (int j = 0; j < QK8_0 * 4; j++) {
- int src_offset = (j / (4 * blck_size_interleave)) * blck_size_interleave;
- int src_id = (j % (4 * blck_size_interleave)) / blck_size_interleave;
- src_offset += (j % blck_size_interleave);
-
- float x0 = srcv[src_id][src_offset] * id[src_id];
- y[i].qs[j] = roundf(x0);
- }
- }
-#endif
-}
-
-void quantize_mat_q8_0(const float * restrict x, void * restrict vy, int64_t nrow, int64_t n_per_row, int64_t blck_size_interleave) {
- assert(nrow == 4);
- UNUSED(nrow);
- if (blck_size_interleave == 4) {
- quantize_q8_0_4x4(x, vy, n_per_row);
- } else if (blck_size_interleave == 8) {
- quantize_q8_0_4x8(x, vy, n_per_row);
- } else {
- assert(false);
- }
-}
-
-static size_t quantize_q4_0_nr_bl(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, int nrows_interleaved, int blck_size_interleave) {
- assert(n_per_row % QK4_0 == 0);
- const int nb = n_per_row / QK4_0;
-
- void * out_ptr = NULL;
- if (nrows_interleaved == 8) {
- out_ptr = (block_q4_0x8 *) dst;
- }
- else if (nrows_interleaved == 4) {
- out_ptr = (block_q4_0x4 *) dst;
- }
- assert(nrows_interleaved <= 8);
- block_q4_0 dst_tmp[8];
-
- for (int b = 0; b < (nrow * n_per_row); b += nrows_interleaved * n_per_row) {
-
- for (int64_t x = 0; x < nb; x++) {
-
- for (int i = 0; i < nrows_interleaved; i++ ) {
- quantize_row_q4_0_ref(src + b + i * n_per_row + x * QK4_0, (block_q4_0 *) dst_tmp + i, QK4_0);
- }
-
- if (nrows_interleaved == 8) {
- *(block_q4_0x8 *) out_ptr = make_block_q4_0x8(dst_tmp, blck_size_interleave, 0x88);
- out_ptr = (block_q4_0x8 *) out_ptr + 1;
- }
- else if (nrows_interleaved == 4) {
- *(block_q4_0x4 *) out_ptr = make_block_q4_0x4(dst_tmp, blck_size_interleave, 0x88);
- out_ptr = (block_q4_0x4 *) out_ptr + 1;
- }
- }
- }
-
- return ((nrow * n_per_row) / QK4_0 * sizeof(block_q4_0));
-}
-
-size_t quantize_q4_0_4x4(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
- if (!quant_weights) {
- return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 4);
- }
- else {
- assert(false);
- return 0;
- }
-}
-
-size_t quantize_q4_0_4x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
- if (!quant_weights) {
- return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 4, 8);
- }
- else {
- assert(false);
- return 0;
- }
-}
-
-size_t quantize_q4_0_8x8(const float * restrict src, void * restrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
- if (!quant_weights) {
- return quantize_q4_0_nr_bl(src, dst, nrow, n_per_row, 8, 8);
- }
- else {
- assert(false);
- return 0;
- }
-}
-
-void ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 4;
- const int blocklen = 4;
-
- assert (n % qk == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
- GGML_ASSERT(!(ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) &&
- "__ARM_NEON and __ARM_FEATURE_MATMUL_INT8 defined, use the Q4_0_4_8 quantization format for optimal performance");
-#elif defined(__ARM_NEON) && defined(__aarch64__) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
-
- __asm__ __volatile__(
- "movi v31.16b, #0x4\n"
- "movi v30.16b, #0xf0\n"
- "add %x[b_ptr], %x[b_ptr], #0x8\n"
- "1:" // Column loop
- "add x22, %x[a_ptr], #0x2\n"
- "movi v29.16b, #0x0\n"
- "mov x21, %x[nb]\n"
- "2:" // Block loop
- "ldr q28, [%x[b_ptr], #0x0]\n"
- "ldr q27, [x22, #0x0]\n"
- "movi v26.4s, #0x0\n"
- "sub x20, x22, #0x2\n"
- "ldr q25, [x22, #0x10]\n"
- "ldr q24, [%x[b_ptr], #0x10]\n"
- "sub x21, x21, #0x1\n"
- "add x22, x22, #0x22\n"
- "ldr q23, [%x[b_ptr], #0x20]\n"
- "ldr q22, [%x[b_ptr], #0x30]\n"
- "ld1r { v21.8h }, [x20]\n"
- "ldr q20, [%x[b_ptr], #-0x8]\n"
- "sshl v16.16b, v28.16b, v31.16b\n"
- "and v28.16b, v28.16b, v30.16b\n"
- "sshl v19.16b, v24.16b, v31.16b\n"
- "and v24.16b, v24.16b, v30.16b\n"
- "add %x[b_ptr], %x[b_ptr], #0x48\n"
- "sshl v18.16b, v23.16b, v31.16b\n"
- "and v23.16b, v23.16b, v30.16b\n"
- ".inst 0x4f9be21a // sdot v26.4s, v16.16b, v27.4b[0]\n"
- "sshl v17.16b, v22.16b, v31.16b\n"
- "and v22.16b, v22.16b, v30.16b\n"
- "fcvtl v21.4s, v21.4h\n"
- "fcvtl v16.4s, v20.4h\n"
- ".inst 0x4f99e39a // sdot v26.4s, v28.16b, v25.4b[0]\n"
- "fmul v16.4s, v16.4s, v21.4s\n"
- ".inst 0x4fbbe27a // sdot v26.4s, v19.16b, v27.4b[1]\n"
- ".inst 0x4fb9e31a // sdot v26.4s, v24.16b, v25.4b[1]\n"
- ".inst 0x4f9bea5a // sdot v26.4s, v18.16b, v27.4b[2]\n"
- ".inst 0x4f99eafa // sdot v26.4s, v23.16b, v25.4b[2]\n"
- ".inst 0x4fbbea3a // sdot v26.4s, v17.16b, v27.4b[3]\n"
- ".inst 0x4fb9eada // sdot v26.4s, v22.16b, v25.4b[3]\n"
- "scvtf v26.4s, v26.4s, #0x4\n"
- "fmla v29.4s, v26.4s, v16.4s\n"
- "cbnz x21, 2b\n"
- "sub %x[nc], %x[nc], #0x4\n"
- "str q29, [%x[res_ptr], #0x0]\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "cbnz %x[nc], 1b\n"
- : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
- : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
- : "memory", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22"
- );
-#else
- float sumf[4];
- int sumi;
-
- const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
-
- for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
- }
- sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
- }
- }
- }
- for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
- }
-#endif
-}
-
-void ggml_gemv_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 4;
- const int blocklen = 8;
-
- assert (n % qk == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
-
- __asm__ __volatile__(
- "movi v2.16b, #0x4\n"
- "movi v1.16b, #0xf0\n"
- "add %x[b_ptr], %x[b_ptr], #0x8\n"
- "1:" // Column loop
- "add x23, %x[a_ptr], #0x2\n"
- "movi v0.16b, #0x0\n"
- "mov x22, %x[nb]\n"
- "2:" // Block loop
- "ldr q31, [%x[b_ptr], #0x0]\n"
- "ldr q30, [%x[b_ptr], #0x10]\n"
- "mov x21, x23\n"
- "movi v29.4s, #0x0\n"
- "ldr q28, [%x[b_ptr], #0x20]\n"
- "ldr q27, [%x[b_ptr], #0x30]\n"
- "movi v26.4s, #0x0\n"
- "sub x20, x23, #0x2\n"
- "ld1r { v25.8h }, [x20]\n"
- "ldr q24, [%x[b_ptr], #-0x8]\n"
- "sub x22, x22, #0x1\n"
- "add x23, x23, #0x22\n"
- "ld1r { v23.2d }, [x21], #0x8\n"
- "sshl v22.16b, v31.16b, v2.16b\n"
- "sshl v16.16b, v30.16b, v2.16b\n"
- "add %x[b_ptr], %x[b_ptr], #0x48\n"
- "ld1r { v21.2d }, [x21], #0x8\n"
- "sshl v20.16b, v28.16b, v2.16b\n"
- "sshl v19.16b, v27.16b, v2.16b\n"
- "ld1r { v18.2d }, [x21], #0x8\n"
- "ld1r { v17.2d }, [x21], #0x8\n"
- "and v31.16b, v31.16b, v1.16b\n"
- "and v30.16b, v30.16b, v1.16b\n"
- ".inst 0x4e9796dd // sdot v29.4s, v22.16b, v23.16b\n"
- ".inst 0x4e97961a // sdot v26.4s, v16.16b, v23.16b\n"
- "and v28.16b, v28.16b, v1.16b\n"
- "and v27.16b, v27.16b, v1.16b\n"
- "fcvtl v25.4s, v25.4h\n"
- "fcvtl v16.4s, v24.4h\n"
- ".inst 0x4e95969d // sdot v29.4s, v20.16b, v21.16b\n"
- ".inst 0x4e95967a // sdot v26.4s, v19.16b, v21.16b\n"
- "fmul v16.4s, v16.4s, v25.4s\n"
- ".inst 0x4e9297fd // sdot v29.4s, v31.16b, v18.16b\n"
- ".inst 0x4e9297da // sdot v26.4s, v30.16b, v18.16b\n"
- ".inst 0x4e91979d // sdot v29.4s, v28.16b, v17.16b\n"
- ".inst 0x4e91977a // sdot v26.4s, v27.16b, v17.16b\n"
- "addp v29.4s, v29.4s, v26.4s\n"
- "scvtf v29.4s, v29.4s, #0x4\n"
- "fmla v0.4s, v29.4s, v16.4s\n"
- "cbnz x22, 2b\n"
- "sub %x[nc], %x[nc], #0x4\n"
- "str q0, [%x[res_ptr], #0x0]\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "cbnz %x[nc], 1b\n"
- : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
- : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
- : "memory", "v0", "v1", "v2", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x20", "x21", "x22", "x23"
- );
-#elif defined(__ARM_NEON) && defined(__aarch64__)
- GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
- "performance");
-#else
- float sumf[4];
- int sumi;
-
- const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
-
- for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
- }
- sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
- }
- }
- }
- for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
- }
-#endif
-}
-
-void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 8;
- const int blocklen = 8;
-
- assert (n % qk == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- if (svcntw() == 8) {
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
-
- __asm__ __volatile__(
- "ptrue p0.b\n"
- "add %x[b_ptr], %x[b_ptr], #0x10\n"
- "1:" // Column loop
- "add x22, %x[a_ptr], #0x2\n"
- "mov z31.b, #0x0\n"
- "mov x21, %x[nb]\n"
- "2:" // Block loop
- "ld1b { z30.b }, p0/Z, [%x[b_ptr]]\n"
- "ld1b { z29.b }, p0/Z, [%x[b_ptr], #1, MUL VL]\n"
- "mov z28.s, #0x0\n"
- "mov z27.s, #0x0\n"
- "ld1rd { z26.d }, p0/Z, [x22]\n"
- "ld1b { z25.b }, p0/Z, [%x[b_ptr], #2, MUL VL]\n"
- "sub x20, x22, #0x2\n"
- "sub x21, x21, #0x1\n"
- "ld1b { z24.b }, p0/Z, [%x[b_ptr], #3, MUL VL]\n"
- "ld1rd { z23.d }, p0/Z, [x22, #8]\n"
- "lsl z22.b, z30.b, #0x4\n"
- "lsl z16.b, z29.b, #0x4\n"
- "and z30.b, z30.b, #0xf0\n"
- "and z29.b, z29.b, #0xf0\n"
- "ld1rd { z21.d }, p0/Z, [x22, #16]\n"
- "ld1rd { z20.d }, p0/Z, [x22, #24]\n"
- "lsl z19.b, z25.b, #0x4\n"
- "and z25.b, z25.b, #0xf0\n"
- "ld1rh { z17.h }, p0/Z, [x20]\n"
- "ld1h { z18.s }, p0/Z, [%x[b_ptr], #-1, MUL VL]\n"
- "sdot z28.s, z22.b, z26.b\n"
- "sdot z27.s, z16.b, z26.b\n"
- "lsl z16.b, z24.b, #0x4\n"
- "add x22, x22, #0x22\n"
- "and z24.b, z24.b, #0xf0\n"
- "add %x[b_ptr], %x[b_ptr], #0x90\n"
- "fcvt z17.s, p0/m, z17.h\n"
- "fcvt z18.s, p0/m, z18.h\n"
- "sdot z28.s, z19.b, z23.b\n"
- "sdot z27.s, z16.b, z23.b\n"
- "fmul z18.s, z18.s, z17.s\n"
- "sdot z28.s, z30.b, z21.b\n"
- "sdot z27.s, z29.b, z21.b\n"
- "sdot z28.s, z25.b, z20.b\n"
- "sdot z27.s, z24.b, z20.b\n"
- "uzp1 z17.s, z28.s, z27.s\n"
- "uzp2 z16.s, z28.s, z27.s\n"
- "add z17.s, z17.s, z16.s\n"
- "asr z17.s, z17.s, #0x4\n"
- "scvtf z17.s, p0/m, z17.s\n"
- "fmla z31.s, p0/M, z17.s, z18.s\n"
- "cbnz x21, 2b\n"
- "sub %x[nc], %x[nc], #0x8\n"
- "st1w { z31.s }, p0, [%x[res_ptr]]\n"
- "add %x[res_ptr], %x[res_ptr], #0x20\n"
- "cbnz %x[nc], 1b\n"
- : [b_ptr] "+&r" (b_ptr), [res_ptr] "+&r" (res_ptr), [nc] "+&r" (nc)
- : [a_ptr] "r" (a_ptr), [nb] "r" (nb)
- : "memory", "p0", "x20", "x21", "x22", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
- );
- return;
- }
- else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
- "performance");
- }
- else if (ggml_cpu_has_neon()) {
- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
- "quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
- GGML_ASSERT(ggml_cpu_has_sve() &&
- "__ARM_FEATURE_SVE not defined, use the Q4_0_4_8 quantization format for optimal performance");
-#elif defined(__ARM_NEON) && defined(__aarch64__)
- GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
- "performance");
-#else
- float sumf[8];
- int sumi;
-
- const block_q8_0 * a_ptr = (const block_q8_0 *) vy;
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);
-
- for (int j = 0; j < ncols_interleaved; j++) sumf[j] = 0.0;
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * blocklen + i]) + (v1 * a_ptr[l].qs[k * blocklen + i + qk / 2])) >> 4;
- }
- sumf[j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d);
- }
- }
- }
- for (int j = 0; j < ncols_interleaved; j++) s[x * ncols_interleaved + j] = sumf[j];
- }
-#endif
-}
-
-void ggml_gemm_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 4;
- const int blocklen = 4;
-
- assert (n % qk == 0);
- assert (nr % 4 == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
- GGML_ASSERT(!(ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) &&
- "__ARM_NEON and __ARM_FEATURE_MATMUL_INT8 defined, use the Q4_0_4_8 quantization format for optimal performance");
-#elif defined(__ARM_NEON) && defined(__aarch64__) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
- size_t res_stride = bs * sizeof(float);
-
- __asm__ __volatile__(
- "mov x10, %x[nr]\n"
- "mov x9, #0x88\n"
- "cmp x10, #0x10\n"
- "mul x9, %x[nb], x9\n"
- "blt 4f\n"
- "1:" // Row loop
- "add x28, %x[b_ptr], #0x8\n"
- "mov x27, %x[nc]\n"
- "add x26, %x[res_ptr], %x[res_stride], LSL #4\n"
- "2:" // Column loop
- "add x25, %x[a_ptr], #0x8\n"
- "movi v15.16b, #0x0\n"
- "movi v19.16b, #0x0\n"
- "mov x24, %x[nb]\n"
- "add x23, x25, x9\n"
- "movi v18.16b, #0x0\n"
- "movi v14.16b, #0x0\n"
- "add x22, x23, x9\n"
- "movi v11.16b, #0x0\n"
- "movi v13.16b, #0x0\n"
- "add x21, x22, x9\n"
- "movi v23.16b, #0x0\n"
- "movi v16.16b, #0x0\n"
- "movi v25.16b, #0x0\n"
- "movi v7.16b, #0x0\n"
- "movi v0.16b, #0x0\n"
- "movi v4.16b, #0x0\n"
- "movi v5.16b, #0x0\n"
- "movi v21.16b, #0x0\n"
- "movi v8.16b, #0x0\n"
- "movi v1.16b, #0x0\n"
- "3:" // Block loop
- "ldr q3, [x28, #0x0]\n"
- "ldr q31, [x25, #0x0]\n"
- "movi v28.16b, #0x4\n"
- "movi v10.4s, #0x0\n"
- "ldr q22, [x28, #0x10]\n"
- "ldr q6, [x25, #0x10]\n"
- "movi v29.4s, #0x0\n"
- "movi v9.4s, #0x0\n"
- "ldr q27, [x28, #0x20]\n"
- "ldr q30, [x28, #0x30]\n"
- "movi v20.4s, #0x0\n"
- "movi v24.16b, #0xf0\n"
- "ldr d2, [x25, #-0x8]\n"
- "ldr d26, [x23, #-0x8]\n"
- "sshl v12.16b, v3.16b, v28.16b\n"
- "sub x20, x28, #0x8\n"
- "ldr d17, [x20, #0x0]\n"
- "and v3.16b, v3.16b, v24.16b\n"
- "subs x24, x24, #0x1\n"
- "add x28, x28, #0x48\n"
- ".inst 0x4f9fe18a // sdot v10.4s, v12.16b, v31.4b[0]\n"
- ".inst 0x4fbfe19d // sdot v29.4s, v12.16b, v31.4b[1]\n"
- ".inst 0x4f9fe989 // sdot v9.4s, v12.16b, v31.4b[2]\n"
- ".inst 0x4fbfe994 // sdot v20.4s, v12.16b, v31.4b[3]\n"
- "sshl v31.16b, v22.16b, v28.16b\n"
- "and v22.16b, v22.16b, v24.16b\n"
- "fcvtl v17.4s, v17.4h\n"
- "fcvtl v2.4s, v2.4h\n"
- "fcvtl v26.4s, v26.4h\n"
- ".inst 0x4f86e3ea // sdot v10.4s, v31.16b, v6.4b[0]\n"
- ".inst 0x4fa6e3fd // sdot v29.4s, v31.16b, v6.4b[1]\n"
- ".inst 0x4f86ebe9 // sdot v9.4s, v31.16b, v6.4b[2]\n"
- ".inst 0x4fa6ebf4 // sdot v20.4s, v31.16b, v6.4b[3]\n"
- "sshl v6.16b, v27.16b, v28.16b\n"
- "sshl v28.16b, v30.16b, v28.16b\n"
- "and v27.16b, v27.16b, v24.16b\n"
- "and v30.16b, v30.16b, v24.16b\n"
- "ldr q24, [x25, #0x20]\n"
- ".inst 0x4f98e0ca // sdot v10.4s, v6.16b, v24.4b[0]\n"
- ".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\n"
- ".inst 0x4f98e8c9 // sdot v9.4s, v6.16b, v24.4b[2]\n"
- ".inst 0x4fb8e8d4 // sdot v20.4s, v6.16b, v24.4b[3]\n"
- "ldr q24, [x25, #0x30]\n"
- ".inst 0x4f98e38a // sdot v10.4s, v28.16b, v24.4b[0]\n"
- ".inst 0x4fb8e39d // sdot v29.4s, v28.16b, v24.4b[1]\n"
- ".inst 0x4f98eb89 // sdot v9.4s, v28.16b, v24.4b[2]\n"
- ".inst 0x4fb8eb94 // sdot v20.4s, v28.16b, v24.4b[3]\n"
- "ldr q24, [x25, #0x40]\n"
- ".inst 0x4f98e06a // sdot v10.4s, v3.16b, v24.4b[0]\n"
- ".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\n"
- ".inst 0x4f98e869 // sdot v9.4s, v3.16b, v24.4b[2]\n"
- ".inst 0x4fb8e874 // sdot v20.4s, v3.16b, v24.4b[3]\n"
- "ldr q24, [x25, #0x50]\n"
- ".inst 0x4f98e2ca // sdot v10.4s, v22.16b, v24.4b[0]\n"
- ".inst 0x4fb8e2dd // sdot v29.4s, v22.16b, v24.4b[1]\n"
- ".inst 0x4f98eac9 // sdot v9.4s, v22.16b, v24.4b[2]\n"
- ".inst 0x4fb8ead4 // sdot v20.4s, v22.16b, v24.4b[3]\n"
- "ldr q24, [x25, #0x60]\n"
- ".inst 0x4f98e36a // sdot v10.4s, v27.16b, v24.4b[0]\n"
- ".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\n"
- ".inst 0x4f98eb69 // sdot v9.4s, v27.16b, v24.4b[2]\n"
- ".inst 0x4fb8eb74 // sdot v20.4s, v27.16b, v24.4b[3]\n"
- "ldr q24, [x25, #0x70]\n"
- "add x25, x25, #0x88\n"
- ".inst 0x4f98e3ca // sdot v10.4s, v30.16b, v24.4b[0]\n"
- ".inst 0x4fb8e3dd // sdot v29.4s, v30.16b, v24.4b[1]\n"
- ".inst 0x4f98ebc9 // sdot v9.4s, v30.16b, v24.4b[2]\n"
- ".inst 0x4fb8ebd4 // sdot v20.4s, v30.16b, v24.4b[3]\n"
- "fmul v24.4s, v17.4s, v2.s[0]\n"
- "scvtf v10.4s, v10.4s, #0x4\n"
- "scvtf v29.4s, v29.4s, #0x4\n"
- "scvtf v9.4s, v9.4s, #0x4\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "fmla v15.4s, v10.4s, v24.4s\n"
- "ldr q24, [x23, #0x0]\n"
- "fmul v10.4s, v17.4s, v2.s[1]\n"
- "fmla v19.4s, v29.4s, v10.4s\n"
- "ldr q10, [x23, #0x10]\n"
- "fmul v29.4s, v17.4s, v2.s[2]\n"
- "fmul v2.4s, v17.4s, v2.s[3]\n"
- "fmla v18.4s, v9.4s, v29.4s\n"
- "movi v9.4s, #0x0\n"
- "movi v29.4s, #0x0\n"
- ".inst 0x4f98e189 // sdot v9.4s, v12.16b, v24.4b[0]\n"
- ".inst 0x4fb8e19d // sdot v29.4s, v12.16b, v24.4b[1]\n"
- "fmla v14.4s, v20.4s, v2.4s\n"
- "movi v20.4s, #0x0\n"
- "movi v2.4s, #0x0\n"
- ".inst 0x4f98e994 // sdot v20.4s, v12.16b, v24.4b[2]\n"
- ".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\n"
- "ldr q24, [x23, #0x20]\n"
- ".inst 0x4f8ae3e9 // sdot v9.4s, v31.16b, v10.4b[0]\n"
- ".inst 0x4faae3fd // sdot v29.4s, v31.16b, v10.4b[1]\n"
- ".inst 0x4f8aebf4 // sdot v20.4s, v31.16b, v10.4b[2]\n"
- ".inst 0x4faaebe2 // sdot v2.4s, v31.16b, v10.4b[3]\n"
- "ldr q10, [x23, #0x30]\n"
- ".inst 0x4f98e0c9 // sdot v9.4s, v6.16b, v24.4b[0]\n"
- ".inst 0x4fb8e0dd // sdot v29.4s, v6.16b, v24.4b[1]\n"
- ".inst 0x4f98e8d4 // sdot v20.4s, v6.16b, v24.4b[2]\n"
- ".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\n"
- "ldr q24, [x23, #0x40]\n"
- ".inst 0x4f8ae389 // sdot v9.4s, v28.16b, v10.4b[0]\n"
- ".inst 0x4faae39d // sdot v29.4s, v28.16b, v10.4b[1]\n"
- ".inst 0x4f8aeb94 // sdot v20.4s, v28.16b, v10.4b[2]\n"
- ".inst 0x4faaeb82 // sdot v2.4s, v28.16b, v10.4b[3]\n"
- "ldr q10, [x23, #0x50]\n"
- ".inst 0x4f98e069 // sdot v9.4s, v3.16b, v24.4b[0]\n"
- ".inst 0x4fb8e07d // sdot v29.4s, v3.16b, v24.4b[1]\n"
- ".inst 0x4f98e874 // sdot v20.4s, v3.16b, v24.4b[2]\n"
- ".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\n"
- "ldr q24, [x23, #0x60]\n"
- ".inst 0x4f8ae2c9 // sdot v9.4s, v22.16b, v10.4b[0]\n"
- ".inst 0x4faae2dd // sdot v29.4s, v22.16b, v10.4b[1]\n"
- ".inst 0x4f8aead4 // sdot v20.4s, v22.16b, v10.4b[2]\n"
- ".inst 0x4faaeac2 // sdot v2.4s, v22.16b, v10.4b[3]\n"
- "ldr q10, [x23, #0x70]\n"
- "add x23, x23, #0x88\n"
- ".inst 0x4f98e369 // sdot v9.4s, v27.16b, v24.4b[0]\n"
- ".inst 0x4fb8e37d // sdot v29.4s, v27.16b, v24.4b[1]\n"
- ".inst 0x4f98eb74 // sdot v20.4s, v27.16b, v24.4b[2]\n"
- ".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\n"
- "ldr q24, [x22, #0x0]\n"
- ".inst 0x4f8ae3c9 // sdot v9.4s, v30.16b, v10.4b[0]\n"
- ".inst 0x4faae3dd // sdot v29.4s, v30.16b, v10.4b[1]\n"
- ".inst 0x4f8aebd4 // sdot v20.4s, v30.16b, v10.4b[2]\n"
- ".inst 0x4faaebc2 // sdot v2.4s, v30.16b, v10.4b[3]\n"
- "fmul v10.4s, v17.4s, v26.s[0]\n"
- "scvtf v9.4s, v9.4s, #0x4\n"
- "scvtf v29.4s, v29.4s, #0x4\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "scvtf v2.4s, v2.4s, #0x4\n"
- "fmla v11.4s, v9.4s, v10.4s\n"
- "ldr q9, [x22, #0x10]\n"
- "fmul v10.4s, v17.4s, v26.s[1]\n"
- "fmla v13.4s, v29.4s, v10.4s\n"
- "ldr d29, [x22, #-0x8]\n"
- "fmul v10.4s, v17.4s, v26.s[2]\n"
- "fmul v26.4s, v17.4s, v26.s[3]\n"
- "fcvtl v29.4s, v29.4h\n"
- "fmla v23.4s, v20.4s, v10.4s\n"
- "movi v20.4s, #0x0\n"
- "movi v10.4s, #0x0\n"
- "fmla v16.4s, v2.4s, v26.4s\n"
- "movi v26.4s, #0x0\n"
- "movi v2.4s, #0x0\n"
- ".inst 0x4f98e194 // sdot v20.4s, v12.16b, v24.4b[0]\n"
- ".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\n"
- ".inst 0x4f98e99a // sdot v26.4s, v12.16b, v24.4b[2]\n"
- ".inst 0x4fb8e982 // sdot v2.4s, v12.16b, v24.4b[3]\n"
- "ldr q24, [x22, #0x20]\n"
- ".inst 0x4f89e3f4 // sdot v20.4s, v31.16b, v9.4b[0]\n"
- ".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\n"
- ".inst 0x4f89ebfa // sdot v26.4s, v31.16b, v9.4b[2]\n"
- ".inst 0x4fa9ebe2 // sdot v2.4s, v31.16b, v9.4b[3]\n"
- "ldr q9, [x22, #0x30]\n"
- ".inst 0x4f98e0d4 // sdot v20.4s, v6.16b, v24.4b[0]\n"
- ".inst 0x4fb8e0ca // sdot v10.4s, v6.16b, v24.4b[1]\n"
- ".inst 0x4f98e8da // sdot v26.4s, v6.16b, v24.4b[2]\n"
- ".inst 0x4fb8e8c2 // sdot v2.4s, v6.16b, v24.4b[3]\n"
- "ldr q24, [x22, #0x40]\n"
- ".inst 0x4f89e394 // sdot v20.4s, v28.16b, v9.4b[0]\n"
- ".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\n"
- ".inst 0x4f89eb9a // sdot v26.4s, v28.16b, v9.4b[2]\n"
- ".inst 0x4fa9eb82 // sdot v2.4s, v28.16b, v9.4b[3]\n"
- "ldr q9, [x22, #0x50]\n"
- ".inst 0x4f98e074 // sdot v20.4s, v3.16b, v24.4b[0]\n"
- ".inst 0x4fb8e06a // sdot v10.4s, v3.16b, v24.4b[1]\n"
- ".inst 0x4f98e87a // sdot v26.4s, v3.16b, v24.4b[2]\n"
- ".inst 0x4fb8e862 // sdot v2.4s, v3.16b, v24.4b[3]\n"
- "ldr q24, [x22, #0x60]\n"
- ".inst 0x4f89e2d4 // sdot v20.4s, v22.16b, v9.4b[0]\n"
- ".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\n"
- ".inst 0x4f89eada // sdot v26.4s, v22.16b, v9.4b[2]\n"
- ".inst 0x4fa9eac2 // sdot v2.4s, v22.16b, v9.4b[3]\n"
- "ldr q9, [x22, #0x70]\n"
- "add x22, x22, #0x88\n"
- ".inst 0x4f98e374 // sdot v20.4s, v27.16b, v24.4b[0]\n"
- ".inst 0x4fb8e36a // sdot v10.4s, v27.16b, v24.4b[1]\n"
- ".inst 0x4f98eb7a // sdot v26.4s, v27.16b, v24.4b[2]\n"
- ".inst 0x4fb8eb62 // sdot v2.4s, v27.16b, v24.4b[3]\n"
- "ldr q24, [x21, #0x0]\n"
- ".inst 0x4f89e3d4 // sdot v20.4s, v30.16b, v9.4b[0]\n"
- ".inst 0x4fa9e3ca // sdot v10.4s, v30.16b, v9.4b[1]\n"
- ".inst 0x4f89ebda // sdot v26.4s, v30.16b, v9.4b[2]\n"
- ".inst 0x4fa9ebc2 // sdot v2.4s, v30.16b, v9.4b[3]\n"
- "fmul v9.4s, v17.4s, v29.s[0]\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "scvtf v10.4s, v10.4s, #0x4\n"
- "scvtf v26.4s, v26.4s, #0x4\n"
- "scvtf v2.4s, v2.4s, #0x4\n"
- "fmla v25.4s, v20.4s, v9.4s\n"
- "ldr q9, [x21, #0x10]\n"
- "fmul v20.4s, v17.4s, v29.s[1]\n"
- "fmla v7.4s, v10.4s, v20.4s\n"
- "ldr d20, [x21, #-0x8]\n"
- "fmul v10.4s, v17.4s, v29.s[2]\n"
- "fmul v29.4s, v17.4s, v29.s[3]\n"
- "fcvtl v20.4s, v20.4h\n"
- "fmla v0.4s, v26.4s, v10.4s\n"
- "movi v26.4s, #0x0\n"
- "movi v10.4s, #0x0\n"
- "fmla v4.4s, v2.4s, v29.4s\n"
- "movi v2.4s, #0x0\n"
- "movi v29.4s, #0x0\n"
- ".inst 0x4f98e19a // sdot v26.4s, v12.16b, v24.4b[0]\n"
- ".inst 0x4fb8e18a // sdot v10.4s, v12.16b, v24.4b[1]\n"
- ".inst 0x4f98e982 // sdot v2.4s, v12.16b, v24.4b[2]\n"
- ".inst 0x4fb8e99d // sdot v29.4s, v12.16b, v24.4b[3]\n"
- "ldr q12, [x21, #0x20]\n"
- "fmul v24.4s, v17.4s, v20.s[0]\n"
- ".inst 0x4f89e3fa // sdot v26.4s, v31.16b, v9.4b[0]\n"
- ".inst 0x4fa9e3ea // sdot v10.4s, v31.16b, v9.4b[1]\n"
- ".inst 0x4f89ebe2 // sdot v2.4s, v31.16b, v9.4b[2]\n"
- ".inst 0x4fa9ebfd // sdot v29.4s, v31.16b, v9.4b[3]\n"
- "ldr q9, [x21, #0x30]\n"
- "fmul v31.4s, v17.4s, v20.s[1]\n"
- ".inst 0x4f8ce0da // sdot v26.4s, v6.16b, v12.4b[0]\n"
- ".inst 0x4face0ca // sdot v10.4s, v6.16b, v12.4b[1]\n"
- ".inst 0x4f8ce8c2 // sdot v2.4s, v6.16b, v12.4b[2]\n"
- ".inst 0x4face8dd // sdot v29.4s, v6.16b, v12.4b[3]\n"
- "ldr q12, [x21, #0x40]\n"
- "fmul v6.4s, v17.4s, v20.s[2]\n"
- "fmul v20.4s, v17.4s, v20.s[3]\n"
- ".inst 0x4f89e39a // sdot v26.4s, v28.16b, v9.4b[0]\n"
- ".inst 0x4fa9e38a // sdot v10.4s, v28.16b, v9.4b[1]\n"
- ".inst 0x4f89eb82 // sdot v2.4s, v28.16b, v9.4b[2]\n"
- ".inst 0x4fa9eb9d // sdot v29.4s, v28.16b, v9.4b[3]\n"
- "ldr q9, [x21, #0x50]\n"
- ".inst 0x4f8ce07a // sdot v26.4s, v3.16b, v12.4b[0]\n"
- ".inst 0x4face06a // sdot v10.4s, v3.16b, v12.4b[1]\n"
- ".inst 0x4f8ce862 // sdot v2.4s, v3.16b, v12.4b[2]\n"
- ".inst 0x4face87d // sdot v29.4s, v3.16b, v12.4b[3]\n"
- "ldr q12, [x21, #0x60]\n"
- ".inst 0x4f89e2da // sdot v26.4s, v22.16b, v9.4b[0]\n"
- ".inst 0x4fa9e2ca // sdot v10.4s, v22.16b, v9.4b[1]\n"
- ".inst 0x4f89eac2 // sdot v2.4s, v22.16b, v9.4b[2]\n"
- ".inst 0x4fa9eadd // sdot v29.4s, v22.16b, v9.4b[3]\n"
- "ldr q17, [x21, #0x70]\n"
- "add x21, x21, #0x88\n"
- ".inst 0x4f8ce37a // sdot v26.4s, v27.16b, v12.4b[0]\n"
- ".inst 0x4face36a // sdot v10.4s, v27.16b, v12.4b[1]\n"
- ".inst 0x4f8ceb62 // sdot v2.4s, v27.16b, v12.4b[2]\n"
- ".inst 0x4faceb7d // sdot v29.4s, v27.16b, v12.4b[3]\n"
- ".inst 0x4f91e3da // sdot v26.4s, v30.16b, v17.4b[0]\n"
- ".inst 0x4fb1e3ca // sdot v10.4s, v30.16b, v17.4b[1]\n"
- ".inst 0x4f91ebc2 // sdot v2.4s, v30.16b, v17.4b[2]\n"
- ".inst 0x4fb1ebdd // sdot v29.4s, v30.16b, v17.4b[3]\n"
- "scvtf v26.4s, v26.4s, #0x4\n"
- "scvtf v10.4s, v10.4s, #0x4\n"
- "fmla v5.4s, v26.4s, v24.4s\n"
- "scvtf v2.4s, v2.4s, #0x4\n"
- "scvtf v29.4s, v29.4s, #0x4\n"
- "fmla v21.4s, v10.4s, v31.4s\n"
- "fmla v8.4s, v2.4s, v6.4s\n"
- "fmla v1.4s, v29.4s, v20.4s\n"
- "bgt 3b\n"
- "mov x20, %x[res_ptr]\n"
- "subs x27, x27, #0x4\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "str q15, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q19, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q18, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q14, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q11, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q13, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q23, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q16, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q25, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q7, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q0, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q4, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q5, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q21, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q8, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q1, [x20, #0x0]\n"
- "bne 2b\n"
- "mov x20, #0x4\n"
- "sub x10, x10, #0x10\n"
- "cmp x10, #0x10\n"
- "mov %x[res_ptr], x26\n"
- "madd %x[a_ptr], x20, x9, %x[a_ptr]\n"
- "bge 1b\n"
- "4:" // Row loop skip
- "cbz x10, 9f\n"
- "5:" // Row tail: Row loop
- "add x24, %x[b_ptr], #0x8\n"
- "mov x23, %x[nc]\n"
- "add x22, %x[res_ptr], %x[res_stride], LSL #2\n"
- "6:" // Row tail: Column loop
- "movi v15.16b, #0x0\n"
- "movi v19.16b, #0x0\n"
- "add x25, %x[a_ptr], #0x8\n"
- "mov x21, %x[nb]\n"
- "movi v18.16b, #0x0\n"
- "movi v14.16b, #0x0\n"
- "7:" // Row tail: Block loop
- "ldr q7, [x24, #0x0]\n"
- "ldr q5, [x25, #0x0]\n"
- "movi v9.16b, #0x4\n"
- "movi v4.4s, #0x0\n"
- "ldr q3, [x24, #0x10]\n"
- "ldr q2, [x25, #0x10]\n"
- "movi v1.4s, #0x0\n"
- "movi v0.4s, #0x0\n"
- "ldr q13, [x24, #0x20]\n"
- "ldr q31, [x25, #0x20]\n"
- "movi v30.4s, #0x0\n"
- "movi v29.16b, #0xf0\n"
- "ldr q28, [x24, #0x30]\n"
- "ldr q27, [x25, #0x30]\n"
- "sshl v20.16b, v7.16b, v9.16b\n"
- "sub x20, x24, #0x8\n"
- "ldr q26, [x25, #0x40]\n"
- "ldr q25, [x25, #0x50]\n"
- "sshl v17.16b, v3.16b, v9.16b\n"
- "and v7.16b, v7.16b, v29.16b\n"
- "ldr q24, [x25, #0x60]\n"
- "ldr q16, [x25, #0x70]\n"
- "sshl v22.16b, v13.16b, v9.16b\n"
- "and v3.16b, v3.16b, v29.16b\n"
- "ldr d21, [x20, #0x0]\n"
- "ldr d12, [x25, #-0x8]\n"
- ".inst 0x4f85e284 // sdot v4.4s, v20.16b, v5.4b[0]\n"
- ".inst 0x4fa5e281 // sdot v1.4s, v20.16b, v5.4b[1]\n"
- ".inst 0x4f85ea80 // sdot v0.4s, v20.16b, v5.4b[2]\n"
- ".inst 0x4fa5ea9e // sdot v30.4s, v20.16b, v5.4b[3]\n"
- "sshl v9.16b, v28.16b, v9.16b\n"
- "subs x21, x21, #0x1\n"
- "and v13.16b, v13.16b, v29.16b\n"
- "and v28.16b, v28.16b, v29.16b\n"
- "add x25, x25, #0x88\n"
- "add x24, x24, #0x48\n"
- "fcvtl v21.4s, v21.4h\n"
- "fcvtl v12.4s, v12.4h\n"
- ".inst 0x4f82e224 // sdot v4.4s, v17.16b, v2.4b[0]\n"
- ".inst 0x4fa2e221 // sdot v1.4s, v17.16b, v2.4b[1]\n"
- ".inst 0x4f82ea20 // sdot v0.4s, v17.16b, v2.4b[2]\n"
- ".inst 0x4fa2ea3e // sdot v30.4s, v17.16b, v2.4b[3]\n"
- "fmul v11.4s, v21.4s, v12.s[0]\n"
- "fmul v23.4s, v21.4s, v12.s[1]\n"
- "fmul v17.4s, v21.4s, v12.s[2]\n"
- ".inst 0x4f9fe2c4 // sdot v4.4s, v22.16b, v31.4b[0]\n"
- "fmul v6.4s, v21.4s, v12.s[3]\n"
- ".inst 0x4fbfe2c1 // sdot v1.4s, v22.16b, v31.4b[1]\n"
- ".inst 0x4f9feac0 // sdot v0.4s, v22.16b, v31.4b[2]\n"
- ".inst 0x4fbfeade // sdot v30.4s, v22.16b, v31.4b[3]\n"
- ".inst 0x4f9be124 // sdot v4.4s, v9.16b, v27.4b[0]\n"
- ".inst 0x4fbbe121 // sdot v1.4s, v9.16b, v27.4b[1]\n"
- ".inst 0x4f9be920 // sdot v0.4s, v9.16b, v27.4b[2]\n"
- ".inst 0x4fbbe93e // sdot v30.4s, v9.16b, v27.4b[3]\n"
- ".inst 0x4f9ae0e4 // sdot v4.4s, v7.16b, v26.4b[0]\n"
- ".inst 0x4fbae0e1 // sdot v1.4s, v7.16b, v26.4b[1]\n"
- ".inst 0x4f9ae8e0 // sdot v0.4s, v7.16b, v26.4b[2]\n"
- ".inst 0x4fbae8fe // sdot v30.4s, v7.16b, v26.4b[3]\n"
- ".inst 0x4f99e064 // sdot v4.4s, v3.16b, v25.4b[0]\n"
- ".inst 0x4fb9e061 // sdot v1.4s, v3.16b, v25.4b[1]\n"
- ".inst 0x4f99e860 // sdot v0.4s, v3.16b, v25.4b[2]\n"
- ".inst 0x4fb9e87e // sdot v30.4s, v3.16b, v25.4b[3]\n"
- ".inst 0x4f98e1a4 // sdot v4.4s, v13.16b, v24.4b[0]\n"
- ".inst 0x4fb8e1a1 // sdot v1.4s, v13.16b, v24.4b[1]\n"
- ".inst 0x4f98e9a0 // sdot v0.4s, v13.16b, v24.4b[2]\n"
- ".inst 0x4fb8e9be // sdot v30.4s, v13.16b, v24.4b[3]\n"
- ".inst 0x4f90e384 // sdot v4.4s, v28.16b, v16.4b[0]\n"
- ".inst 0x4fb0e381 // sdot v1.4s, v28.16b, v16.4b[1]\n"
- ".inst 0x4f90eb80 // sdot v0.4s, v28.16b, v16.4b[2]\n"
- ".inst 0x4fb0eb9e // sdot v30.4s, v28.16b, v16.4b[3]\n"
- "scvtf v4.4s, v4.4s, #0x4\n"
- "scvtf v1.4s, v1.4s, #0x4\n"
- "scvtf v0.4s, v0.4s, #0x4\n"
- "fmla v15.4s, v4.4s, v11.4s\n"
- "scvtf v30.4s, v30.4s, #0x4\n"
- "fmla v19.4s, v1.4s, v23.4s\n"
- "fmla v18.4s, v0.4s, v17.4s\n"
- "fmla v14.4s, v30.4s, v6.4s\n"
- "bgt 7b\n"
- "mov x20, %x[res_ptr]\n"
- "cmp x10, #0x1\n"
- "str q15, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x10, #0x2\n"
- "str q19, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x10, #0x3\n"
- "str q18, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "str q14, [x20, #0x0]\n"
- "8:" // Row tail: Accumulator store skip
- "subs x23, x23, #0x4\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "bne 6b\n"
- "subs x10, x10, #0x4\n"
- "add %x[a_ptr], %x[a_ptr], x9\n"
- "mov %x[res_ptr], x22\n"
- "bgt 5b\n"
- "9:" // Row tail: Row loop skip
- : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
- : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
- );
-#else
- float sumf[4][4];
- int sumi;
-
- for (int y = 0; y < nr / 4; y++) {
- const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
- }
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
- (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
- }
- sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
- }
- }
- }
- }
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++)
- s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
- }
- }
- }
-#endif
-}
-
-void ggml_gemm_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 4;
- const int blocklen = 8;
-
- assert (n % qk == 0);
- assert (nr % 4 == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
- size_t res_stride = bs * sizeof(float);
-
- __asm__ __volatile__(
- "mov x10, %x[nr]\n"
- "mov x9, #0x88\n"
- "cmp x10, #0x10\n"
- "mul x9, %x[nb], x9\n"
- "blt 4f\n"
- "1:" // Row loop
- "add x28, %x[b_ptr], #0x8\n"
- "mov x27, %x[nc]\n"
- "add x26, %x[res_ptr], %x[res_stride], LSL #4\n"
- "2:" // Column loop
- "add x25, %x[a_ptr], #0x8\n"
- "movi v2.16b, #0x0\n"
- "movi v10.16b, #0x0\n"
- "mov x24, %x[nb]\n"
- "add x23, x25, x9\n"
- "movi v12.16b, #0x0\n"
- "movi v28.16b, #0x0\n"
- "add x22, x23, x9\n"
- "movi v11.16b, #0x0\n"
- "movi v13.16b, #0x0\n"
- "add x21, x22, x9\n"
- "movi v22.16b, #0x0\n"
- "movi v23.16b, #0x0\n"
- "movi v25.16b, #0x0\n"
- "movi v5.16b, #0x0\n"
- "movi v7.16b, #0x0\n"
- "movi v4.16b, #0x0\n"
- "movi v6.16b, #0x0\n"
- "movi v30.16b, #0x0\n"
- "movi v24.16b, #0x0\n"
- "movi v14.16b, #0x0\n"
- "3:" // Block loop
- "ldr q21, [x28, #0x0]\n"
- "ldr q16, [x28, #0x10]\n"
- "movi v1.16b, #0x4\n"
- "movi v19.4s, #0x0\n"
- "ldr q27, [x25, #0x0]\n"
- "ldr q15, [x25, #0x10]\n"
- "movi v26.4s, #0x0\n"
- "movi v18.4s, #0x0\n"
- "ldr q29, [x28, #0x20]\n"
- "ldr q3, [x28, #0x30]\n"
- "movi v17.4s, #0x0\n"
- "movi v0.16b, #0xf0\n"
- "ldr d20, [x25, #-0x8]\n"
- "ldr d9, [x23, #-0x8]\n"
- "sshl v8.16b, v21.16b, v1.16b\n"
- "sshl v31.16b, v16.16b, v1.16b\n"
- "and v21.16b, v21.16b, v0.16b\n"
- "and v16.16b, v16.16b, v0.16b\n"
- "sub x20, x28, #0x8\n"
- "subs x24, x24, #0x1\n"
- "add x28, x28, #0x48\n"
- ".inst 0x4e88a773 // smmla v19.4s, v27.16b, v8.16b\n"
- ".inst 0x4e9fa77a // smmla v26.4s, v27.16b, v31.16b\n"
- "ldr q27, [x25, #0x20]\n"
- ".inst 0x4e88a5f2 // smmla v18.4s, v15.16b, v8.16b\n"
- ".inst 0x4e9fa5f1 // smmla v17.4s, v15.16b, v31.16b\n"
- "sshl v15.16b, v29.16b, v1.16b\n"
- "sshl v1.16b, v3.16b, v1.16b\n"
- "and v29.16b, v29.16b, v0.16b\n"
- "and v3.16b, v3.16b, v0.16b\n"
- "ldr q0, [x25, #0x30]\n"
- "fcvtl v20.4s, v20.4h\n"
- ".inst 0x4e8fa773 // smmla v19.4s, v27.16b, v15.16b\n"
- "fcvtl v9.4s, v9.4h\n"
- ".inst 0x4e81a77a // smmla v26.4s, v27.16b, v1.16b\n"
- "ldr q27, [x25, #0x40]\n"
- ".inst 0x4e8fa412 // smmla v18.4s, v0.16b, v15.16b\n"
- ".inst 0x4e81a411 // smmla v17.4s, v0.16b, v1.16b\n"
- "ldr q0, [x25, #0x50]\n"
- ".inst 0x4e95a773 // smmla v19.4s, v27.16b, v21.16b\n"
- ".inst 0x4e90a77a // smmla v26.4s, v27.16b, v16.16b\n"
- "ldr q27, [x25, #0x60]\n"
- ".inst 0x4e95a412 // smmla v18.4s, v0.16b, v21.16b\n"
- ".inst 0x4e90a411 // smmla v17.4s, v0.16b, v16.16b\n"
- "ldr q0, [x25, #0x70]\n"
- "add x25, x25, #0x88\n"
- ".inst 0x4e9da773 // smmla v19.4s, v27.16b, v29.16b\n"
- ".inst 0x4e83a77a // smmla v26.4s, v27.16b, v3.16b\n"
- "ldr d27, [x20, #0x0]\n"
- ".inst 0x4e9da412 // smmla v18.4s, v0.16b, v29.16b\n"
- ".inst 0x4e83a411 // smmla v17.4s, v0.16b, v3.16b\n"
- "fcvtl v27.4s, v27.4h\n"
- "uzp1 v0.2d, v19.2d, v26.2d\n"
- "uzp2 v26.2d, v19.2d, v26.2d\n"
- "fmul v19.4s, v27.4s, v20.s[0]\n"
- "scvtf v0.4s, v0.4s, #0x4\n"
- "scvtf v26.4s, v26.4s, #0x4\n"
- "fmla v2.4s, v0.4s, v19.4s\n"
- "ldr q19, [x23, #0x0]\n"
- "uzp1 v0.2d, v18.2d, v17.2d\n"
- "uzp2 v18.2d, v18.2d, v17.2d\n"
- "fmul v17.4s, v27.4s, v20.s[1]\n"
- "scvtf v0.4s, v0.4s, #0x4\n"
- "scvtf v18.4s, v18.4s, #0x4\n"
- "fmla v10.4s, v26.4s, v17.4s\n"
- "ldr q17, [x23, #0x10]\n"
- "fmul v26.4s, v27.4s, v20.s[2]\n"
- "fmul v20.4s, v27.4s, v20.s[3]\n"
- "fmla v12.4s, v0.4s, v26.4s\n"
- "ldr d0, [x22, #-0x8]\n"
- "ldr d26, [x21, #-0x8]\n"
- "fcvtl v0.4s, v0.4h\n"
- "fmla v28.4s, v18.4s, v20.4s\n"
- "movi v20.4s, #0x0\n"
- "movi v18.4s, #0x0\n"
- ".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\n"
- ".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\n"
- "ldr q19, [x23, #0x20]\n"
- "fcvtl v26.4s, v26.4h\n"
- ".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\n"
- ".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\n"
- "ldr q19, [x23, #0x40]\n"
- ".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\n"
- ".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\n"
- "ldr q19, [x23, #0x60]\n"
- ".inst 0x4e9da674 // smmla v20.4s, v19.16b, v29.16b\n"
- ".inst 0x4e83a672 // smmla v18.4s, v19.16b, v3.16b\n"
- "uzp1 v19.2d, v20.2d, v18.2d\n"
- "scvtf v19.4s, v19.4s, #0x4\n"
- "uzp2 v20.2d, v20.2d, v18.2d\n"
- "fmul v18.4s, v27.4s, v9.s[0]\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "fmla v11.4s, v19.4s, v18.4s\n"
- "ldr q18, [x22, #0x0]\n"
- "fmul v19.4s, v27.4s, v9.s[1]\n"
- "fmla v13.4s, v20.4s, v19.4s\n"
- "movi v19.4s, #0x0\n"
- "movi v20.4s, #0x0\n"
- ".inst 0x4e88a633 // smmla v19.4s, v17.16b, v8.16b\n"
- ".inst 0x4e9fa634 // smmla v20.4s, v17.16b, v31.16b\n"
- "ldr q17, [x23, #0x30]\n"
- ".inst 0x4e8fa633 // smmla v19.4s, v17.16b, v15.16b\n"
- ".inst 0x4e81a634 // smmla v20.4s, v17.16b, v1.16b\n"
- "ldr q17, [x23, #0x50]\n"
- ".inst 0x4e95a633 // smmla v19.4s, v17.16b, v21.16b\n"
- ".inst 0x4e90a634 // smmla v20.4s, v17.16b, v16.16b\n"
- "ldr q17, [x23, #0x70]\n"
- "add x23, x23, #0x88\n"
- ".inst 0x4e9da633 // smmla v19.4s, v17.16b, v29.16b\n"
- ".inst 0x4e83a634 // smmla v20.4s, v17.16b, v3.16b\n"
- "uzp1 v17.2d, v19.2d, v20.2d\n"
- "scvtf v17.4s, v17.4s, #0x4\n"
- "uzp2 v20.2d, v19.2d, v20.2d\n"
- "fmul v19.4s, v27.4s, v9.s[2]\n"
- "fmul v9.4s, v27.4s, v9.s[3]\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "fmla v22.4s, v17.4s, v19.4s\n"
- "ldr q17, [x22, #0x10]\n"
- "movi v19.4s, #0x0\n"
- ".inst 0x4e88a653 // smmla v19.4s, v18.16b, v8.16b\n"
- "fmla v23.4s, v20.4s, v9.4s\n"
- "movi v20.4s, #0x0\n"
- "movi v9.4s, #0x0\n"
- ".inst 0x4e9fa654 // smmla v20.4s, v18.16b, v31.16b\n"
- "ldr q18, [x22, #0x20]\n"
- ".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\n"
- ".inst 0x4e8fa653 // smmla v19.4s, v18.16b, v15.16b\n"
- ".inst 0x4e81a654 // smmla v20.4s, v18.16b, v1.16b\n"
- "ldr q18, [x22, #0x40]\n"
- ".inst 0x4e95a653 // smmla v19.4s, v18.16b, v21.16b\n"
- ".inst 0x4e90a654 // smmla v20.4s, v18.16b, v16.16b\n"
- "ldr q18, [x22, #0x60]\n"
- ".inst 0x4e9da653 // smmla v19.4s, v18.16b, v29.16b\n"
- ".inst 0x4e83a654 // smmla v20.4s, v18.16b, v3.16b\n"
- "movi v18.4s, #0x0\n"
- ".inst 0x4e9fa632 // smmla v18.4s, v17.16b, v31.16b\n"
- "ldr q17, [x22, #0x30]\n"
- ".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\n"
- ".inst 0x4e81a632 // smmla v18.4s, v17.16b, v1.16b\n"
- "ldr q17, [x22, #0x50]\n"
- ".inst 0x4e95a629 // smmla v9.4s, v17.16b, v21.16b\n"
- ".inst 0x4e90a632 // smmla v18.4s, v17.16b, v16.16b\n"
- "ldr q17, [x22, #0x70]\n"
- "add x22, x22, #0x88\n"
- ".inst 0x4e9da629 // smmla v9.4s, v17.16b, v29.16b\n"
- ".inst 0x4e83a632 // smmla v18.4s, v17.16b, v3.16b\n"
- "uzp1 v17.2d, v19.2d, v20.2d\n"
- "uzp2 v20.2d, v19.2d, v20.2d\n"
- "fmul v19.4s, v27.4s, v0.s[0]\n"
- "scvtf v17.4s, v17.4s, #0x4\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "fmla v25.4s, v17.4s, v19.4s\n"
- "ldr q19, [x21, #0x0]\n"
- "fmul v17.4s, v27.4s, v0.s[1]\n"
- "fmla v5.4s, v20.4s, v17.4s\n"
- "ldr q17, [x21, #0x10]\n"
- "uzp1 v20.2d, v9.2d, v18.2d\n"
- "uzp2 v9.2d, v9.2d, v18.2d\n"
- "fmul v18.4s, v27.4s, v0.s[2]\n"
- "fmul v0.4s, v27.4s, v0.s[3]\n"
- "scvtf v20.4s, v20.4s, #0x4\n"
- "scvtf v9.4s, v9.4s, #0x4\n"
- "fmla v7.4s, v20.4s, v18.4s\n"
- "movi v20.4s, #0x0\n"
- "movi v18.4s, #0x0\n"
- ".inst 0x4e88a674 // smmla v20.4s, v19.16b, v8.16b\n"
- ".inst 0x4e9fa672 // smmla v18.4s, v19.16b, v31.16b\n"
- "ldr q19, [x21, #0x20]\n"
- "fmla v4.4s, v9.4s, v0.4s\n"
- "movi v9.4s, #0x0\n"
- "movi v0.4s, #0x0\n"
- ".inst 0x4e88a629 // smmla v9.4s, v17.16b, v8.16b\n"
- "fmul v8.4s, v27.4s, v26.s[0]\n"
- ".inst 0x4e9fa620 // smmla v0.4s, v17.16b, v31.16b\n"
- "ldr q17, [x21, #0x30]\n"
- ".inst 0x4e8fa674 // smmla v20.4s, v19.16b, v15.16b\n"
- "fmul v31.4s, v27.4s, v26.s[1]\n"
- ".inst 0x4e81a672 // smmla v18.4s, v19.16b, v1.16b\n"
- "ldr q19, [x21, #0x40]\n"
- ".inst 0x4e8fa629 // smmla v9.4s, v17.16b, v15.16b\n"
- "fmul v15.4s, v27.4s, v26.s[2]\n"
- "fmul v27.4s, v27.4s, v26.s[3]\n"
- ".inst 0x4e81a620 // smmla v0.4s, v17.16b, v1.16b\n"
- "ldr q1, [x21, #0x50]\n"
- ".inst 0x4e95a674 // smmla v20.4s, v19.16b, v21.16b\n"
- ".inst 0x4e90a672 // smmla v18.4s, v19.16b, v16.16b\n"
- "ldr q26, [x21, #0x60]\n"
- ".inst 0x4e95a429 // smmla v9.4s, v1.16b, v21.16b\n"
- ".inst 0x4e90a420 // smmla v0.4s, v1.16b, v16.16b\n"
- "ldr q21, [x21, #0x70]\n"
- "add x21, x21, #0x88\n"
- ".inst 0x4e9da754 // smmla v20.4s, v26.16b, v29.16b\n"
- ".inst 0x4e83a752 // smmla v18.4s, v26.16b, v3.16b\n"
- ".inst 0x4e9da6a9 // smmla v9.4s, v21.16b, v29.16b\n"
- ".inst 0x4e83a6a0 // smmla v0.4s, v21.16b, v3.16b\n"
- "uzp1 v29.2d, v20.2d, v18.2d\n"
- "uzp2 v21.2d, v20.2d, v18.2d\n"
- "scvtf v29.4s, v29.4s, #0x4\n"
- "uzp1 v18.2d, v9.2d, v0.2d\n"
- "uzp2 v16.2d, v9.2d, v0.2d\n"
- "scvtf v21.4s, v21.4s, #0x4\n"
- "fmla v6.4s, v29.4s, v8.4s\n"
- "scvtf v18.4s, v18.4s, #0x4\n"
- "scvtf v16.4s, v16.4s, #0x4\n"
- "fmla v30.4s, v21.4s, v31.4s\n"
- "fmla v24.4s, v18.4s, v15.4s\n"
- "fmla v14.4s, v16.4s, v27.4s\n"
- "bgt 3b\n"
- "mov x20, %x[res_ptr]\n"
- "subs x27, x27, #0x4\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "str q2, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q10, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q12, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q28, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q11, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q13, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q22, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q23, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q25, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q5, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q7, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q4, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q6, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q30, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q24, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "str q14, [x20, #0x0]\n"
- "bne 2b\n"
- "mov x20, #0x4\n"
- "sub x10, x10, #0x10\n"
- "cmp x10, #0x10\n"
- "mov %x[res_ptr], x26\n"
- "madd %x[a_ptr], x20, x9, %x[a_ptr]\n"
- "bge 1b\n"
- "4:" // Row loop skip
- "cbz x10, 9f\n"
- "5:" // Row tail: Row loop
- "add x24, %x[b_ptr], #0x8\n"
- "mov x23, %x[nc]\n"
- "add x22, %x[res_ptr], %x[res_stride], LSL #2\n"
- "6:" // Row tail: Column loop
- "movi v2.16b, #0x0\n"
- "movi v10.16b, #0x0\n"
- "add x25, %x[a_ptr], #0x8\n"
- "mov x21, %x[nb]\n"
- "movi v12.16b, #0x0\n"
- "movi v28.16b, #0x0\n"
- "7:" // Row tail: Block loop
- "ldr q6, [x24, #0x0]\n"
- "ldr q5, [x24, #0x10]\n"
- "movi v17.16b, #0x4\n"
- "movi v8.4s, #0x0\n"
- "ldr q4, [x25, #0x0]\n"
- "ldr q13, [x25, #0x10]\n"
- "movi v27.4s, #0x0\n"
- "movi v0.4s, #0x0\n"
- "ldr q31, [x24, #0x20]\n"
- "ldr q14, [x24, #0x30]\n"
- "movi v29.4s, #0x0\n"
- "movi v22.16b, #0xf0\n"
- "ldr q11, [x25, #0x20]\n"
- "ldr q23, [x25, #0x30]\n"
- "sshl v21.16b, v6.16b, v17.16b\n"
- "sshl v16.16b, v5.16b, v17.16b\n"
- "ldr q20, [x25, #0x40]\n"
- "ldr q26, [x25, #0x50]\n"
- "and v6.16b, v6.16b, v22.16b\n"
- "and v5.16b, v5.16b, v22.16b\n"
- "ldr q25, [x25, #0x60]\n"
- "ldr q3, [x25, #0x70]\n"
- "sshl v19.16b, v31.16b, v17.16b\n"
- "sshl v18.16b, v14.16b, v17.16b\n"
- "ldr d17, [x25, #-0x8]\n"
- ".inst 0x4e95a488 // smmla v8.4s, v4.16b, v21.16b\n"
- ".inst 0x4e90a49b // smmla v27.4s, v4.16b, v16.16b\n"
- "and v31.16b, v31.16b, v22.16b\n"
- ".inst 0x4e95a5a0 // smmla v0.4s, v13.16b, v21.16b\n"
- ".inst 0x4e90a5bd // smmla v29.4s, v13.16b, v16.16b\n"
- "and v14.16b, v14.16b, v22.16b\n"
- "sub x20, x24, #0x8\n"
- "ldr d16, [x20, #0x0]\n"
- "subs x21, x21, #0x1\n"
- "add x25, x25, #0x88\n"
- "fcvtl v17.4s, v17.4h\n"
- "add x24, x24, #0x48\n"
- ".inst 0x4e93a568 // smmla v8.4s, v11.16b, v19.16b\n"
- ".inst 0x4e92a57b // smmla v27.4s, v11.16b, v18.16b\n"
- ".inst 0x4e93a6e0 // smmla v0.4s, v23.16b, v19.16b\n"
- ".inst 0x4e92a6fd // smmla v29.4s, v23.16b, v18.16b\n"
- "fcvtl v16.4s, v16.4h\n"
- ".inst 0x4e86a688 // smmla v8.4s, v20.16b, v6.16b\n"
- ".inst 0x4e85a69b // smmla v27.4s, v20.16b, v5.16b\n"
- "fmul v23.4s, v16.4s, v17.s[0]\n"
- "fmul v21.4s, v16.4s, v17.s[1]\n"
- "fmul v1.4s, v16.4s, v17.s[2]\n"
- "fmul v20.4s, v16.4s, v17.s[3]\n"
- ".inst 0x4e86a740 // smmla v0.4s, v26.16b, v6.16b\n"
- ".inst 0x4e85a75d // smmla v29.4s, v26.16b, v5.16b\n"
- ".inst 0x4e9fa728 // smmla v8.4s, v25.16b, v31.16b\n"
- ".inst 0x4e8ea73b // smmla v27.4s, v25.16b, v14.16b\n"
- ".inst 0x4e9fa460 // smmla v0.4s, v3.16b, v31.16b\n"
- ".inst 0x4e8ea47d // smmla v29.4s, v3.16b, v14.16b\n"
- "uzp1 v19.2d, v8.2d, v27.2d\n"
- "uzp2 v18.2d, v8.2d, v27.2d\n"
- "scvtf v19.4s, v19.4s, #0x4\n"
- "uzp1 v17.2d, v0.2d, v29.2d\n"
- "uzp2 v16.2d, v0.2d, v29.2d\n"
- "scvtf v18.4s, v18.4s, #0x4\n"
- "fmla v2.4s, v19.4s, v23.4s\n"
- "scvtf v17.4s, v17.4s, #0x4\n"
- "scvtf v16.4s, v16.4s, #0x4\n"
- "fmla v10.4s, v18.4s, v21.4s\n"
- "fmla v12.4s, v17.4s, v1.4s\n"
- "fmla v28.4s, v16.4s, v20.4s\n"
- "bgt 7b\n"
- "mov x20, %x[res_ptr]\n"
- "cmp x10, #0x1\n"
- "str q2, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x10, #0x2\n"
- "str q10, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x10, #0x3\n"
- "str q12, [x20, #0x0]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "str q28, [x20, #0x0]\n"
- "8:" // Row tail: Accumulator store skip
- "subs x23, x23, #0x4\n"
- "add %x[res_ptr], %x[res_ptr], #0x10\n"
- "bne 6b\n"
- "subs x10, x10, #0x4\n"
- "add %x[a_ptr], %x[a_ptr], x9\n"
- "mov %x[res_ptr], x22\n"
- "bgt 5b\n"
- "9:" // Row tail: Row loop skip
- : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
- : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
- : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
- );
-#elif defined(__ARM_NEON) && defined(__aarch64__)
- GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
- "performance");
-#else
- float sumf[4][4];
- int sumi;
-
- for (int y = 0; y < nr / 4; y++) {
- const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x4 * b_ptr = (const block_q4_0x4 *) vx + (x * nb);
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
- }
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
- (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
- }
- sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
- }
- }
- }
- }
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++)
- s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
- }
- }
- }
-#endif
-}
-
-void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void * restrict vx, const void * restrict vy, int nr, int nc) {
- const int qk = QK8_0;
- const int nb = n / qk;
- const int ncols_interleaved = 8;
- const int blocklen = 8;
-
- assert (n % qk == 0);
- assert (nr % 4 == 0);
- assert (nc % ncols_interleaved == 0);
-
- UNUSED(s);
- UNUSED(bs);
- UNUSED(vx);
- UNUSED(vy);
- UNUSED(nr);
- UNUSED(nc);
- UNUSED(nb);
- UNUSED(ncols_interleaved);
- UNUSED(blocklen);
-
-#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- if (svcntw() == 8) {
- const void * b_ptr = vx;
- const void * a_ptr = vy;
- float * res_ptr = s;
- size_t res_stride = bs * sizeof(float);
-
- __asm__ __volatile__(
- "mov x20, #0x4\n"
- "mov x13, %x[nr]\n"
- "mov z28.s, #-0x4\n"
- "mov x12, #0x88\n"
- "ptrue p1.b\n"
- "whilelt p0.s, XZR, x20\n"
- "cmp x13, #0x10\n"
- "mul x12, %x[nb], x12\n"
- "blt 4f\n"
- "1:" // Row loop
- "add x11, %x[b_ptr], #0x10\n"
- "mov x10, %x[nc]\n"
- "add x9, %x[res_ptr], %x[res_stride], LSL #4\n"
- "2:" // Column loop
- "add x28, %x[a_ptr], #0x8\n"
- "mov z24.b, #0x0\n"
- "mov z15.b, #0x0\n"
- "mov x27, %x[nb]\n"
- "add x26, x28, x12\n"
- "mov z12.b, #0x0\n"
- "mov z0.b, #0x0\n"
- "add x25, x26, x12\n"
- "mov z13.b, #0x0\n"
- "mov z1.b, #0x0\n"
- "add x24, x25, x12\n"
- "mov z20.b, #0x0\n"
- "mov z25.b, #0x0\n"
- "mov z11.b, #0x0\n"
- "mov z16.b, #0x0\n"
- "mov z19.b, #0x0\n"
- "mov z26.b, #0x0\n"
- "mov z8.b, #0x0\n"
- "mov z29.b, #0x0\n"
- "mov z27.b, #0x0\n"
- "mov z10.b, #0x0\n"
- "3:" // Block loop
- "ld1b { z30.b }, p1/Z, [x11]\n"
- "ld1b { z21.b }, p1/Z, [x11, #1, MUL VL]\n"
- "mov z18.s, #0x0\n"
- "mov z7.s, #0x0\n"
- "ld1rqb { z3.b }, p1/Z, [x28]\n"
- "ld1rqb { z5.b }, p1/Z, [x28, #16]\n"
- "mov z9.s, #0x0\n"
- "mov z22.s, #0x0\n"
- "ld1b { z4.b }, p1/Z, [x11, #2, MUL VL]\n"
- "ld1b { z17.b }, p1/Z, [x11, #3, MUL VL]\n"
- "sub x20, x11, #0x10\n"
- "sub x23, x28, #0x8\n"
- "lsl z31.b, z30.b, #0x4\n"
- "lsl z6.b, z21.b, #0x4\n"
- "ld1h { z23.s }, p1/Z, [x20]\n"
- "sub x22, x26, #0x8\n"
- "and z30.b, z30.b, #0xf0\n"
- "and z21.b, z21.b, #0xf0\n"
- "sub x21, x25, #0x8\n"
- "sub x20, x24, #0x8\n"
- "lsl z14.b, z4.b, #0x4\n"
- "lsl z2.b, z17.b, #0x4\n"
- "subs x27, x27, #0x1\n"
- "add x11, x11, #0x90\n"
- ".inst 0x451f9872 // smmla z18.s, z3.b, z31.b\n"
- ".inst 0x45069867 // smmla z7.s, z3.b, z6.b\n"
- "ld1rqb { z3.b }, p1/Z, [x28, #32]\n"
- "and z4.b, z4.b, #0xf0\n"
- ".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\n"
- ".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\n"
- "ld1rqb { z5.b }, p1/Z, [x28, #48]\n"
- "and z17.b, z17.b, #0xf0\n"
- "fcvt z23.s, p1/m, z23.h\n"
- ".inst 0x450e9872 // smmla z18.s, z3.b, z14.b\n"
- ".inst 0x45029867 // smmla z7.s, z3.b, z2.b\n"
- "ld1rqb { z3.b }, p1/Z, [x28, #64]\n"
- ".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\n"
- ".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\n"
- "ld1rqb { z5.b }, p1/Z, [x28, #80]\n"
- "fscale z23.s, p1/m, z23.s, z28.s\n"
- ".inst 0x451e9872 // smmla z18.s, z3.b, z30.b\n"
- ".inst 0x45159867 // smmla z7.s, z3.b, z21.b\n"
- "ld1rqb { z3.b }, p1/Z, [x28, #96]\n"
- ".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\n"
- ".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\n"
- "ld1rqb { z5.b }, p1/Z, [x28, #112]\n"
- "add x28, x28, #0x88\n"
- ".inst 0x45049872 // smmla z18.s, z3.b, z4.b\n"
- ".inst 0x45119867 // smmla z7.s, z3.b, z17.b\n"
- "ld1h { z3.s }, p0/Z, [x23]\n"
- ".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\n"
- ".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\n"
- "fcvt z3.s, p1/m, z3.h\n"
- "uzp1 z5.d, z18.d, z7.d\n"
- "uzp2 z18.d, z18.d, z7.d\n"
- "mov z3.q, z3.q[0]\n"
- "uzp1 z7.d, z9.d, z22.d\n"
- "uzp2 z22.d, z9.d, z22.d\n"
- "fmul z9.s, z23.s, z3.s[0]\n"
- "scvtf z5.s, p1/m, z5.s\n"
- "scvtf z18.s, p1/m, z18.s\n"
- "scvtf z7.s, p1/m, z7.s\n"
- "scvtf z22.s, p1/m, z22.s\n"
- "fmla z24.s, p1/M, z5.s, z9.s\n"
- "ld1rqb { z5.b }, p1/Z, [x26]\n"
- "fmul z9.s, z23.s, z3.s[1]\n"
- "fmla z15.s, p1/M, z18.s, z9.s\n"
- "ld1rqb { z18.b }, p1/Z, [x26, #16]\n"
- "fmul z9.s, z23.s, z3.s[2]\n"
- "fmul z3.s, z23.s, z3.s[3]\n"
- "fmla z12.s, p1/M, z7.s, z9.s\n"
- "mov z9.s, #0x0\n"
- "ld1h { z7.s }, p0/Z, [x22]\n"
- ".inst 0x451f98a9 // smmla z9.s, z5.b, z31.b\n"
- "fmla z0.s, p1/M, z22.s, z3.s\n"
- "mov z22.s, #0x0\n"
- "ld1h { z3.s }, p0/Z, [x21]\n"
- ".inst 0x450698b6 // smmla z22.s, z5.b, z6.b\n"
- "ld1rqb { z5.b }, p1/Z, [x26, #32]\n"
- "fcvt z7.s, p1/m, z7.h\n"
- "fcvt z3.s, p1/m, z3.h\n"
- ".inst 0x450e98a9 // smmla z9.s, z5.b, z14.b\n"
- ".inst 0x450298b6 // smmla z22.s, z5.b, z2.b\n"
- "ld1rqb { z5.b }, p1/Z, [x26, #64]\n"
- "mov z7.q, z7.q[0]\n"
- "mov z3.q, z3.q[0]\n"
- ".inst 0x451e98a9 // smmla z9.s, z5.b, z30.b\n"
- ".inst 0x451598b6 // smmla z22.s, z5.b, z21.b\n"
- "ld1rqb { z5.b }, p1/Z, [x26, #96]\n"
- ".inst 0x450498a9 // smmla z9.s, z5.b, z4.b\n"
- ".inst 0x451198b6 // smmla z22.s, z5.b, z17.b\n"
- "uzp1 z5.d, z9.d, z22.d\n"
- "scvtf z5.s, p1/m, z5.s\n"
- "uzp2 z22.d, z9.d, z22.d\n"
- "fmul z9.s, z23.s, z7.s[0]\n"
- "scvtf z22.s, p1/m, z22.s\n"
- "fmla z13.s, p1/M, z5.s, z9.s\n"
- "ld1rqb { z9.b }, p1/Z, [x25]\n"
- "fmul z5.s, z23.s, z7.s[1]\n"
- "fmla z1.s, p1/M, z22.s, z5.s\n"
- "mov z5.s, #0x0\n"
- "mov z22.s, #0x0\n"
- ".inst 0x451f9a45 // smmla z5.s, z18.b, z31.b\n"
- ".inst 0x45069a56 // smmla z22.s, z18.b, z6.b\n"
- "ld1rqb { z18.b }, p1/Z, [x26, #48]\n"
- ".inst 0x450e9a45 // smmla z5.s, z18.b, z14.b\n"
- ".inst 0x45029a56 // smmla z22.s, z18.b, z2.b\n"
- "ld1rqb { z18.b }, p1/Z, [x26, #80]\n"
- ".inst 0x451e9a45 // smmla z5.s, z18.b, z30.b\n"
- ".inst 0x45159a56 // smmla z22.s, z18.b, z21.b\n"
- "ld1rqb { z18.b }, p1/Z, [x26, #112]\n"
- "add x26, x26, #0x88\n"
- ".inst 0x45049a45 // smmla z5.s, z18.b, z4.b\n"
- ".inst 0x45119a56 // smmla z22.s, z18.b, z17.b\n"
- "uzp1 z18.d, z5.d, z22.d\n"
- "scvtf z18.s, p1/m, z18.s\n"
- "uzp2 z22.d, z5.d, z22.d\n"
- "fmul z5.s, z23.s, z7.s[2]\n"
- "fmul z7.s, z23.s, z7.s[3]\n"
- "scvtf z22.s, p1/m, z22.s\n"
- "fmla z20.s, p1/M, z18.s, z5.s\n"
- "ld1rqb { z18.b }, p1/Z, [x25, #16]\n"
- "ld1h { z5.s }, p0/Z, [x20]\n"
- "fcvt z5.s, p1/m, z5.h\n"
- "fmla z25.s, p1/M, z22.s, z7.s\n"
- "mov z22.s, #0x0\n"
- "mov z7.s, #0x0\n"
- ".inst 0x451f9936 // smmla z22.s, z9.b, z31.b\n"
- ".inst 0x45069927 // smmla z7.s, z9.b, z6.b\n"
- "ld1rqb { z9.b }, p1/Z, [x25, #32]\n"
- "mov z5.q, z5.q[0]\n"
- ".inst 0x450e9936 // smmla z22.s, z9.b, z14.b\n"
- ".inst 0x45029927 // smmla z7.s, z9.b, z2.b\n"
- "ld1rqb { z9.b }, p1/Z, [x25, #64]\n"
- ".inst 0x451e9936 // smmla z22.s, z9.b, z30.b\n"
- ".inst 0x45159927 // smmla z7.s, z9.b, z21.b\n"
- "ld1rqb { z9.b }, p1/Z, [x25, #96]\n"
- ".inst 0x45049936 // smmla z22.s, z9.b, z4.b\n"
- ".inst 0x45119927 // smmla z7.s, z9.b, z17.b\n"
- "uzp1 z9.d, z22.d, z7.d\n"
- "scvtf z9.s, p1/m, z9.s\n"
- "uzp2 z22.d, z22.d, z7.d\n"
- "fmul z7.s, z23.s, z3.s[0]\n"
- "scvtf z22.s, p1/m, z22.s\n"
- "fmla z11.s, p1/M, z9.s, z7.s\n"
- "ld1rqb { z9.b }, p1/Z, [x24]\n"
- "fmul z7.s, z23.s, z3.s[1]\n"
- "fmla z16.s, p1/M, z22.s, z7.s\n"
- "mov z22.s, #0x0\n"
- "mov z7.s, #0x0\n"
- ".inst 0x451f9a56 // smmla z22.s, z18.b, z31.b\n"
- ".inst 0x45069a47 // smmla z7.s, z18.b, z6.b\n"
- "ld1rqb { z18.b }, p1/Z, [x25, #48]\n"
- ".inst 0x450e9a56 // smmla z22.s, z18.b, z14.b\n"
- ".inst 0x45029a47 // smmla z7.s, z18.b, z2.b\n"
- "ld1rqb { z18.b }, p1/Z, [x25, #80]\n"
- ".inst 0x451e9a56 // smmla z22.s, z18.b, z30.b\n"
- ".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\n"
- "ld1rqb { z18.b }, p1/Z, [x25, #112]\n"
- "add x25, x25, #0x88\n"
- ".inst 0x45049a56 // smmla z22.s, z18.b, z4.b\n"
- ".inst 0x45119a47 // smmla z7.s, z18.b, z17.b\n"
- "uzp1 z18.d, z22.d, z7.d\n"
- "scvtf z18.s, p1/m, z18.s\n"
- "uzp2 z7.d, z22.d, z7.d\n"
- "fmul z22.s, z23.s, z3.s[2]\n"
- "fmul z3.s, z23.s, z3.s[3]\n"
- "scvtf z7.s, p1/m, z7.s\n"
- "fmla z19.s, p1/M, z18.s, z22.s\n"
- "ld1rqb { z18.b }, p1/Z, [x24, #16]\n"
- "fmul z22.s, z23.s, z5.s[0]\n"
- "fmla z26.s, p1/M, z7.s, z3.s\n"
- "mov z3.s, #0x0\n"
- "mov z7.s, #0x0\n"
- ".inst 0x451f9923 // smmla z3.s, z9.b, z31.b\n"
- ".inst 0x45069927 // smmla z7.s, z9.b, z6.b\n"
- "ld1rqb { z9.b }, p1/Z, [x24, #32]\n"
- ".inst 0x450e9923 // smmla z3.s, z9.b, z14.b\n"
- ".inst 0x45029927 // smmla z7.s, z9.b, z2.b\n"
- "mov z9.s, #0x0\n"
- ".inst 0x451f9a49 // smmla z9.s, z18.b, z31.b\n"
- "mov z31.s, #0x0\n"
- ".inst 0x45069a5f // smmla z31.s, z18.b, z6.b\n"
- "ld1rqb { z6.b }, p1/Z, [x24, #48]\n"
- "ld1rqb { z18.b }, p1/Z, [x24, #64]\n"
- ".inst 0x450e98c9 // smmla z9.s, z6.b, z14.b\n"
- "fmul z14.s, z23.s, z5.s[1]\n"
- ".inst 0x450298df // smmla z31.s, z6.b, z2.b\n"
- "ld1rqb { z6.b }, p1/Z, [x24, #80]\n"
- "fmul z2.s, z23.s, z5.s[2]\n"
- "fmul z23.s, z23.s, z5.s[3]\n"
- ".inst 0x451e9a43 // smmla z3.s, z18.b, z30.b\n"
- ".inst 0x45159a47 // smmla z7.s, z18.b, z21.b\n"
- "ld1rqb { z5.b }, p1/Z, [x24, #96]\n"
- ".inst 0x451e98c9 // smmla z9.s, z6.b, z30.b\n"
- ".inst 0x451598df // smmla z31.s, z6.b, z21.b\n"
- "ld1rqb { z18.b }, p1/Z, [x24, #112]\n"
- "add x24, x24, #0x88\n"
- ".inst 0x450498a3 // smmla z3.s, z5.b, z4.b\n"
- ".inst 0x451198a7 // smmla z7.s, z5.b, z17.b\n"
- ".inst 0x45049a49 // smmla z9.s, z18.b, z4.b\n"
- ".inst 0x45119a5f // smmla z31.s, z18.b, z17.b\n"
- "uzp1 z18.d, z3.d, z7.d\n"
- "uzp2 z5.d, z3.d, z7.d\n"
- "scvtf z18.s, p1/m, z18.s\n"
- "uzp1 z6.d, z9.d, z31.d\n"
- "uzp2 z9.d, z9.d, z31.d\n"
- "scvtf z5.s, p1/m, z5.s\n"
- "fmla z8.s, p1/M, z18.s, z22.s\n"
- "scvtf z6.s, p1/m, z6.s\n"
- "scvtf z9.s, p1/m, z9.s\n"
- "fmla z29.s, p1/M, z5.s, z14.s\n"
- "fmla z27.s, p1/M, z6.s, z2.s\n"
- "fmla z10.s, p1/M, z9.s, z23.s\n"
- "bgt 3b\n"
- "mov x20, %x[res_ptr]\n"
- "subs x10, x10, #0x8\n"
- "add %x[res_ptr], %x[res_ptr], #0x20\n"
- "st1w { z24.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z15.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z12.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z0.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z13.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z1.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z20.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z25.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z11.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z16.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z19.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z26.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z8.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z29.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z27.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "st1w { z10.s }, p1, [x20]\n"
- "bne 2b\n"
- "mov x20, #0x4\n"
- "sub x13, x13, #0x10\n"
- "cmp x13, #0x10\n"
- "mov %x[res_ptr], x9\n"
- "madd %x[a_ptr], x20, x12, %x[a_ptr]\n"
- "bge 1b\n"
- "4:" // Row loop skip
- "cbz x13, 9f\n"
- "5:" // Row tail: Row loop
- "add x25, %x[b_ptr], #0x10\n"
- "mov x24, %x[nc]\n"
- "add x23, %x[res_ptr], %x[res_stride], LSL #2\n"
- "6:" // Row tail: Column loop
- "mov z24.b, #0x0\n"
- "mov z15.b, #0x0\n"
- "add x28, %x[a_ptr], #0x8\n"
- "mov x22, %x[nb]\n"
- "mov z12.b, #0x0\n"
- "mov z0.b, #0x0\n"
- "7:" // Row tail: Block loop
- "ld1b { z3.b }, p1/Z, [x25]\n"
- "ld1b { z6.b }, p1/Z, [x25, #1, MUL VL]\n"
- "mov z2.s, #0x0\n"
- "mov z25.s, #0x0\n"
- "ld1rqb { z26.b }, p1/Z, [x28]\n"
- "ld1rqb { z21.b }, p1/Z, [x28, #16]\n"
- "mov z27.s, #0x0\n"
- "mov z19.s, #0x0\n"
- "ld1b { z29.b }, p1/Z, [x25, #2, MUL VL]\n"
- "ld1b { z16.b }, p1/Z, [x25, #3, MUL VL]\n"
- "sub x21, x25, #0x10\n"
- "sub x20, x28, #0x8\n"
- "lsl z20.b, z3.b, #0x4\n"
- "lsl z4.b, z6.b, #0x4\n"
- "ld1rqb { z10.b }, p1/Z, [x28, #32]\n"
- "ld1rqb { z23.b }, p1/Z, [x28, #48]\n"
- "and z3.b, z3.b, #0xf0\n"
- "and z6.b, z6.b, #0xf0\n"
- "ld1rqb { z11.b }, p1/Z, [x28, #64]\n"
- "ld1rqb { z7.b }, p1/Z, [x28, #80]\n"
- "lsl z8.b, z29.b, #0x4\n"
- "lsl z14.b, z16.b, #0x4\n"
- "ld1rqb { z18.b }, p1/Z, [x28, #96]\n"
- "ld1rqb { z30.b }, p1/Z, [x28, #112]\n"
- ".inst 0x45149b42 // smmla z2.s, z26.b, z20.b\n"
- ".inst 0x45049b59 // smmla z25.s, z26.b, z4.b\n"
- "and z29.b, z29.b, #0xf0\n"
- "ld1h { z17.s }, p1/Z, [x21]\n"
- ".inst 0x45149abb // smmla z27.s, z21.b, z20.b\n"
- ".inst 0x45049ab3 // smmla z19.s, z21.b, z4.b\n"
- "and z16.b, z16.b, #0xf0\n"
- "ld1h { z4.s }, p0/Z, [x20]\n"
- "subs x22, x22, #0x1\n"
- "add x28, x28, #0x88\n"
- "fcvt z17.s, p1/m, z17.h\n"
- "add x25, x25, #0x90\n"
- ".inst 0x45089942 // smmla z2.s, z10.b, z8.b\n"
- ".inst 0x450e9959 // smmla z25.s, z10.b, z14.b\n"
- "fcvt z4.s, p1/m, z4.h\n"
- ".inst 0x45089afb // smmla z27.s, z23.b, z8.b\n"
- ".inst 0x450e9af3 // smmla z19.s, z23.b, z14.b\n"
- "fscale z17.s, p1/m, z17.s, z28.s\n"
- "mov z4.q, z4.q[0]\n"
- ".inst 0x45039962 // smmla z2.s, z11.b, z3.b\n"
- ".inst 0x45069979 // smmla z25.s, z11.b, z6.b\n"
- "fmul z23.s, z17.s, z4.s[0]\n"
- "fmul z9.s, z17.s, z4.s[1]\n"
- "fmul z21.s, z17.s, z4.s[2]\n"
- "fmul z4.s, z17.s, z4.s[3]\n"
- ".inst 0x450398fb // smmla z27.s, z7.b, z3.b\n"
- ".inst 0x450698f3 // smmla z19.s, z7.b, z6.b\n"
- ".inst 0x451d9a42 // smmla z2.s, z18.b, z29.b\n"
- ".inst 0x45109a59 // smmla z25.s, z18.b, z16.b\n"
- ".inst 0x451d9bdb // smmla z27.s, z30.b, z29.b\n"
- ".inst 0x45109bd3 // smmla z19.s, z30.b, z16.b\n"
- "uzp1 z31.d, z2.d, z25.d\n"
- "uzp2 z13.d, z2.d, z25.d\n"
- "scvtf z31.s, p1/m, z31.s\n"
- "uzp1 z17.d, z27.d, z19.d\n"
- "uzp2 z18.d, z27.d, z19.d\n"
- "scvtf z13.s, p1/m, z13.s\n"
- "fmla z24.s, p1/M, z31.s, z23.s\n"
- "scvtf z17.s, p1/m, z17.s\n"
- "scvtf z18.s, p1/m, z18.s\n"
- "fmla z15.s, p1/M, z13.s, z9.s\n"
- "fmla z12.s, p1/M, z17.s, z21.s\n"
- "fmla z0.s, p1/M, z18.s, z4.s\n"
- "bgt 7b\n"
- "mov x20, %x[res_ptr]\n"
- "cmp x13, #0x1\n"
- "st1w { z24.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x13, #0x2\n"
- "st1w { z15.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "cmp x13, #0x3\n"
- "st1w { z12.s }, p1, [x20]\n"
- "add x20, x20, %x[res_stride]\n"
- "ble 8f\n"
- "st1w { z0.s }, p1, [x20]\n"
- "8:" // Row tail: Accumulator store skip
- "subs x24, x24, #0x8\n"
- "add %x[res_ptr], %x[res_ptr], #0x20\n"
- "bne 6b\n"
- "subs x13, x13, #0x4\n"
- "add %x[a_ptr], %x[a_ptr], x12\n"
- "mov %x[res_ptr], x23\n"
- "bgt 5b\n"
- "9:" // Row tail: Row loop skip
- : [a_ptr] "+&r" (a_ptr), [res_ptr] "+&r" (res_ptr)
- : [b_ptr] "r" (b_ptr), [nr] "r" (nr), [nb] "r" (nb), [res_stride] "r" (res_stride), [nc] "r" (nc)
- : "cc", "memory", "p0", "p1", "x9", "x10", "x11", "x12", "x13", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31"
- );
- return;
- }
- else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&
- "__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
- "performance");
- }
- else if (ggml_cpu_has_neon()) {
- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
- "quantization format for optimal performance");
- }
-#endif
-#if defined(__ARM_NEON) && defined(__ARM_FEATURE_MATMUL_INT8)
- GGML_ASSERT(ggml_cpu_has_sve() &&
- "__ARM_FEATURE_SVE not defined, use the Q4_0_4_8 quantization format for optimal performance");
-#elif defined(__ARM_NEON) && defined(__aarch64__)
- GGML_ASSERT((ggml_cpu_has_sve() || ggml_cpu_has_matmul_int8()) &&
- "__ARM_FEATURE_SVE and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 quantization format for optimal "
- "performance");
-#else
- float sumf[4][8];
- int sumi;
-
- for (int y = 0; y < nr / 4; y++) {
- const block_q8_0x4 * a_ptr = (const block_q8_0x4 *) vy + (y * nb);
- for (int x = 0; x < nc / ncols_interleaved; x++) {
- const block_q4_0x8 * b_ptr = (const block_q4_0x8 *) vx + (x * nb);
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) sumf[m][j] = 0.0;
- }
- for (int l = 0; l < nb; l++) {
- for (int k = 0; k < (qk / (2 * blocklen)); k++) {
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++) {
- sumi = 0;
- for (int i = 0; i < blocklen; ++i) {
- const int v0 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] << 4);
- const int v1 = (int8_t) (b_ptr[l].qs[k * ncols_interleaved * blocklen + j * blocklen + i] & 0xF0);
- sumi += ((v0 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i]) +
- (v1 * a_ptr[l].qs[k * 4 * blocklen + m * blocklen + i + qk / 2 * 4])) >> 4;
- }
- sumf[m][j] += sumi * GGML_FP16_TO_FP32(b_ptr[l].d[j]) * GGML_FP16_TO_FP32(a_ptr[l].d[m]);
- }
- }
- }
- }
- for (int m = 0; m < 4; m++) {
- for (int j = 0; j < ncols_interleaved; j++)
- s[(y * 4 + m) * bs + x * ncols_interleaved + j] = sumf[m][j];
- }
- }
- }
-#endif
-}
+++ /dev/null
-/*
- * Copyright (c) 2023-2024 The ggml authors
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include "ggml-cann.h"
-
-#include <acl/acl.h>
-#include <stdarg.h>
-
-#include <cmath>
-#include <cstdio>
-#include <cstring>
-#include <mutex>
-
-#include "ggml-backend-impl.h"
-#include "ggml-cann/aclnn_ops.h"
-#include "ggml-cann/common.h"
-
-#define GGML_COMMON_DECL_C
-
-#include "ggml-common.h"
-
-/**
- * @brief Default logging callback for GGML.
- *
- * This function is the default logging callback that logs messages to stderr.
- *
- * @param level The log level.
- * @param msg The log message.
- * @param user_data User data passed to the callback.
- */
-static void ggml_cann_default_log_callback(enum ggml_log_level level,
- const char* msg, void* user_data) {
- GGML_UNUSED(level);
- GGML_UNUSED(user_data);
- fprintf(stderr, "%s", msg);
-}
-
-ggml_log_callback ggml_cann_log_callback = ggml_cann_default_log_callback;
-void* ggml_cann_log_user_data = NULL;
-
-GGML_API void ggml_backend_cann_log_set_callback(ggml_log_callback log_callback,
- void* user_data) {
- ggml_cann_log_callback = log_callback;
- ggml_cann_log_user_data = user_data;
-}
-
-#define GGML_CANN_LOG_INFO(...) ggml_cann_log(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
-#define GGML_CANN_LOG_WARN(...) ggml_cann_log(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
-#define GGML_CANN_LOG_ERROR(...) \
- ggml_cann_log(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
-
-GGML_ATTRIBUTE_FORMAT(2, 3)
-
-/**
- * @brief Log a message using the current logging callback.
- *
- * This function formats a log message and passes it to the current logging
- * callback.
- *
- * @param level The log level.
- * @param format The format string for the log message.
- * @param ... The arguments for the format string.
- */
-static void ggml_cann_log(enum ggml_log_level level, const char* format, ...) {
- if (ggml_cann_log_callback != NULL) {
- va_list args;
- va_start(args, format);
- char buffer[128];
- int len = vsnprintf(buffer, 128, format, args);
- if (len < 128) {
- ggml_cann_log_callback(level, buffer, ggml_cann_log_user_data);
- } else {
- // vsnprintf adds a null terminator
- std::vector<char> buffer2(len + 1);
- va_end(args);
- va_start(args, format);
- vsnprintf(&buffer2[0], buffer2.size(), format, args);
- ggml_cann_log_callback(level, buffer2.data(),
- ggml_cann_log_user_data);
- }
- va_end(args);
- }
-}
-
-/**
- * @brief Handles CANN errors by printing an error message and aborting.
- *
- * @param stmt The statement that caused the error.
- * @param func The function in which the error occurred.
- * @param file The file in which the error occurred.
- * @param line The line number where the error occurred.
- * @param msg The error message.
- */
-[[noreturn]] void ggml_cann_error(const char* stmt, const char* func,
- const char* file, int line, const char* msg) {
- int32_t id = -1;
- aclrtGetDevice(&id);
-
- GGML_CANN_LOG_ERROR("CANN error: %s\n", msg);
- GGML_CANN_LOG_ERROR(" current device: %d, in function %s at %s:%d\n", id, func,
- file, line);
- GGML_CANN_LOG_ERROR(" %s\n", stmt);
- // abort with GGML_ASSERT to get a stack trace
- GGML_ABORT("CANN error");
-}
-
-/**
- * @brief Sets the device to be used by CANN.
- *
- * @param device The device ID to set.
- */
-void ggml_cann_set_device(const int32_t device) {
- // TODO: uncomment these lines after empty context has fixed.
- // int current_device;
- // ACL_CHECK(aclrtGetDevice(¤t_device));
-
- // if (device == current_device) {
- // return;
- // }
- ACL_CHECK(aclrtSetDevice(device));
-}
-
-/**
- * @brief Retrieves the current device ID.
- *
- * @return The current device ID.
- */
-int32_t ggml_cann_get_device() {
- int32_t id;
- ACL_CHECK(aclrtGetDevice(&id));
- return id;
-}
-
-/**
- * @brief Initialize the CANN device information.
- *
- * This function initializes the CANN device information by obtaining the
- * device count and setting the memory allocation granularity for each device.
- *
- * @return A structure containing the device information.
- */
-static ggml_cann_device_info ggml_cann_init() {
- ggml_cann_device_info info = {};
-
- aclError err = aclrtGetDeviceCount((uint32_t*)&info.device_count);
-
- if (err != ACL_SUCCESS) {
- GGML_CANN_LOG_ERROR("%s: failed to initialize CANN: %s\n",
- __func__, aclGetRecentErrMsg());
- return info;
- }
-
- GGML_ASSERT(info.device_count <= GGML_CANN_MAX_DEVICES);
-
- for (int id = 0; id < info.device_count; ++id) {
- aclrtPhysicalMemProp prop = {};
- prop.handleType = ACL_MEM_HANDLE_TYPE_NONE;
- prop.allocationType = ACL_MEM_ALLOCATION_TYPE_PINNED;
- prop.memAttr = ACL_HBM_MEM_HUGE;
- prop.location.type = ACL_MEM_LOCATION_TYPE_DEVICE;
- prop.location.id = id;
- prop.reserve = 0;
- ACL_CHECK(aclrtMemGetAllocationGranularity(
- &prop, ACL_RT_MEM_ALLOC_GRANULARITY_RECOMMENDED,
- &info.devices[id].vmm_granularity));
- }
-
- // TODO: add more device info later.
- return info;
-}
-
-/**
- * @brief Retrieve the CANN device information.
- *
- * This function returns a reference to a structure containing the CANN device
- * information. The device information is initialized once and reused on
- * subsequent calls.
- *
- * @return A reference to the structure containing the device information.
- */
-const ggml_cann_device_info& ggml_cann_info() {
- static ggml_cann_device_info info = ggml_cann_init();
- return info;
-}
-
-//#define DEBUG_CANN_MALLOC
-/**
- * @brief A pool of CANN buffers(legacy).
- *
- * This class manages a pool of CANN buffers for a specific device.
- */
-struct ggml_cann_pool_leg : public ggml_cann_pool {
- /**
- * @brief The maximum number of buffers in the pool.
- */
- static const int MAX_BUFFERS = 256;
-
- /**
- * @brief The device ID associated with this buffer pool.
- */
- int device;
-
- /**
- * @brief Structure representing a CANN buffer.
- */
- struct ggml_cann_buffer {
- void* ptr = nullptr; ///< Pointer to the buffer memory.
- size_t size = 0; ///< Size of the buffer.
- };
-
- /**
- * @brief Array of CANN buffers in the pool.
- */
- ggml_cann_buffer buffer_pool[MAX_BUFFERS] = {};
-
- /**
- * @brief Total size of all buffers in the pool.
- */
- size_t pool_size = 0;
-
- /**
- * @brief Constructor to initialize the buffer pool for a specific device.
- *
- * @param device The device ID to associate with this buffer pool.
- */
- explicit ggml_cann_pool_leg(int device) : device(device) {}
-
- /**
- * @brief Destructor to free all buffers in the pool.
- */
- ~ggml_cann_pool_leg() {
- ggml_cann_set_device(device);
- for (int i = 0; i < MAX_BUFFERS; ++i) {
- ggml_cann_buffer& b = buffer_pool[i];
- if (b.ptr != nullptr) {
- ACL_CHECK(aclrtFree(b.ptr));
- pool_size -= b.size;
- }
- }
- GGML_ASSERT(pool_size == 0);
- }
-
- /**
- * @brief Allocate a buffer of the given size.
- *
- * @param size The size of the buffer to allocate.
- * @param actual_size A pointer to a variable to receive the actual size of
- * the allocated buffer.
- * @return A pointer to the allocated buffer.
- */
- void* alloc(size_t size, size_t* actual_size) override {
-#ifdef DEBUG_CANN_MALLOC
- int nnz = 0;
- size_t max_size = 0;
-#endif
- size_t best_diff = 1ull << 36;
- int ibest = -1;
- for (int i = 0; i < MAX_BUFFERS; ++i) {
- ggml_cann_buffer& b = buffer_pool[i];
- if (b.ptr != nullptr) {
-#ifdef DEBUG_CANN_MALLOC
- ++nnz;
- if (b.size > max_size) max_size = b.size;
-#endif
- if (b.size >= size) {
- size_t diff = b.size - size;
- if (diff < best_diff) {
- best_diff = diff;
- ibest = i;
- if (!best_diff) {
- void* ptr = b.ptr;
- *actual_size = b.size;
- b.ptr = nullptr;
- b.size = 0;
- return ptr;
- }
- }
- }
- }
- }
- if (ibest >= 0) {
- ggml_cann_buffer& b = buffer_pool[ibest];
- void* ptr = b.ptr;
- *actual_size = b.size;
- b.ptr = nullptr;
- b.size = 0;
- return ptr;
- }
- void* ptr;
- size_t look_ahead_size = (size_t)(1.05 * size);
- look_ahead_size = 256 * ((look_ahead_size + 255) / 256);
- ggml_cann_set_device(device);
- ACL_CHECK(
- aclrtMalloc(&ptr, look_ahead_size, ACL_MEM_MALLOC_HUGE_FIRST));
- *actual_size = look_ahead_size;
- pool_size += look_ahead_size;
-#ifdef DEBUG_CANN_MALLOC
- GGML_CANN_LOG_INFO(
- "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, "
- "requested %u MB\n",
- __func__, device, nnz, (uint32_t)(max_size / 1024 / 1024),
- (uint32_t)(pool_size / 1024 / 1024),
- (uint32_t)(size / 1024 / 1024));
-#endif
- return ptr;
- }
-
- /**
- * @brief Free a buffer and return it to the pool.
- *
- * @param ptr Pointer to the buffer to free.
- * @param size Size of the buffer to free.
- */
- void free(void* ptr, size_t size) override {
- for (int i = 0; i < MAX_BUFFERS; ++i) {
- ggml_cann_buffer& b = buffer_pool[i];
- if (b.ptr == nullptr) {
- b.ptr = ptr;
- b.size = size;
- return;
- }
- }
- // memory should always buffered. these memory may still needed by
- // tasks in stream.
- // TODO, fix me.
- GGML_ABORT("Cann buffer pool full, increase MAX_CANN_BUFFERS\n");
- }
-};
-
-/**
- * @brief A pool of CANN buffers with virtual memory.
- *
- * This class manages a pool of CANN buffers with virtual memory for a specific
- * device.
- */
-struct ggml_cann_pool_vmm : public ggml_cann_pool {
- /**
- * @brief The maximum size of the virtual memory pool (32 GB).
- */
- static const size_t CANN_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB
-
- /**
- * @brief The device ID associated with this buffer pool.
- */
- int device;
-
- /**
- * @brief Pointer to the start of the virtual memory pool.
- */
- void* pool_addr = 0;
-
- /**
- * @brief Amount of virtual memory used in the pool.
- */
- size_t pool_used = 0;
-
- /**
- * @brief Total size of the virtual memory pool.
- */
- size_t pool_size = 0;
-
- /**
- * @brief Allocation granularity for the virtual memory pool.
- */
- size_t granularity;
-
- /**
- * @brief Handles for the physical memory allocated.
- */
- std::vector<aclrtDrvMemHandle> handles;
-
- /**
- * @brief Offsets for the mapped memory regions.
- */
- std::vector<void*> map_offsets;
-
- /**
- * @brief Constructor to initialize the buffer pool with virtual memory for
- * a specific device.
- *
- * @param device The device ID to associate with this buffer pool.
- */
- explicit ggml_cann_pool_vmm(int device)
- : device(device),
- granularity(ggml_cann_info().devices[device].vmm_granularity) {}
-
- /**
- * @brief Destructor to free all buffers in the virtual memory pool.
- */
- ~ggml_cann_pool_vmm() {
- if (pool_addr != 0) {
- for (auto& offset : map_offsets) {
- ACL_CHECK(aclrtUnmapMem(offset));
- }
- for (auto& handle : handles) {
- ACL_CHECK(aclrtFreePhysical(handle));
- }
- ACL_CHECK(aclrtReleaseMemAddress(pool_addr));
- }
- }
-
- /**
- * @brief Allocate a buffer of the given size in the virtual memory pool.
- *
- * @param size The size of the buffer to allocate.
- * @param actual_size A pointer to a variable to receive the actual size of
- * the allocated buffer.
- * @return A pointer to the allocated buffer.
- */
- void* alloc(size_t size, size_t* actual_size) override {
- // round up the allocation size to the alignment to ensure that all
- // allocations are aligned for all data types
- const size_t alignment = 128;
- size = alignment * ((size + alignment - 1) / alignment);
-
- size_t avail = pool_size - pool_used;
-
- if (size > avail) {
- // round up to the next multiple of the granularity
- size_t reserve_size = size - avail;
- reserve_size =
- granularity * ((reserve_size + granularity - 1) / granularity);
-
- GGML_ASSERT(pool_size + reserve_size <= CANN_POOL_VMM_MAX_SIZE);
-
- // allocate more physical memory
- aclrtPhysicalMemProp prop = {};
- prop.handleType = ACL_MEM_HANDLE_TYPE_NONE;
- prop.allocationType = ACL_MEM_ALLOCATION_TYPE_PINNED;
- prop.memAttr = ACL_HBM_MEM_HUGE;
- prop.location.type = ACL_MEM_LOCATION_TYPE_DEVICE;
- prop.location.id = device;
- prop.reserve = 0;
- aclrtDrvMemHandle handle;
- ACL_CHECK(aclrtMallocPhysical(&handle, reserve_size, &prop, 0));
-
- // reserve virtual address space (if not already reserved)
- if (pool_addr == 0) {
- ACL_CHECK(aclrtReserveMemAddress(
- &pool_addr, CANN_POOL_VMM_MAX_SIZE, 0, NULL, 1));
- }
-
- // map at the end of the pool
- ACL_CHECK(aclrtMapMem((char*)pool_addr + pool_size, reserve_size, 0,
- handle, 0));
-
- handles.push_back(handle);
- map_offsets.push_back((char*)pool_addr + pool_size);
-
- // add to the pool
- pool_size += reserve_size;
-
- // GGML_CANN_LOG_INFO("cann pool[%d]: size increased to %llu MB (
- // reserved %llu MB)\n",
- // device, (unsigned long long) (pool_size/1024/1024),
- // (unsigned long long) (reserve_size/1024/1024));
- }
-
- GGML_ASSERT(pool_addr != 0);
-
- void* ptr = (void*)((char*)pool_addr + pool_used);
- *actual_size = size;
- pool_used += size;
-
-#ifdef DEBUG_CANN_MALLOC
- GGML_CANN_LOG_INFO("cann pool[%d]: allocated %llu bytes at %llx\n", device,
- (unsigned long long)size, (unsigned long long)ptr);
-#endif
- return ptr;
- }
-
- /**
- * @brief Free a buffer and return it to the virtual memory pool.
- *
- * @param ptr Pointer to the buffer to free.
- * @param size Size of the buffer to free.
- */
- void free(void* ptr, size_t size) override {
-#ifdef DEBUG_CANN_MALLOC
- GGML_CANN_LOG_INFO("cann pool[%d]: freed %llu bytes at %llx\n", device,
- (unsigned long long)size, (unsigned long long)ptr);
-#endif
-
- pool_used -= size;
-
- // all deallocations must be in reverse order of the allocations
- GGML_ASSERT(ptr == (void*)((char*)pool_addr + pool_used));
- }
-};
-
-/**
- * @brief Create a new CANN pool for a specific device.
- *
- * Factory method to create a new CANN pool object based on the device type.
- *
- * @param device The device ID for which to create the pool.
- * @return A unique pointer to the created CANN pool.
- */
-std::unique_ptr<ggml_cann_pool> ggml_backend_cann_context::new_pool_for_device(
- int device) {
- // return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_leg(device));
- return std::unique_ptr<ggml_cann_pool>(new ggml_cann_pool_vmm(device));
-}
-
-// cann buffer
-/**
- * @brief Context for managing a CANN buffer associated with a specific device.
- *
- * This structure holds information about a CANN buffer, including the device
- * ID, device pointer, and a name derived from GGML_CANN_NAME and the device ID.
- */
-struct ggml_backend_cann_buffer_context {
- int32_t device; ///< The device ID associated with this buffer context.
- void* dev_ptr =
- nullptr; ///< Pointer to the device memory allocated for the buffer.
-
- /**
- * @brief Constructor to initialize the CANN buffer context.
- *
- * @param device The device ID associated with this buffer context.
- * @param dev_ptr Pointer to the device memory allocated for the buffer.
- */
- ggml_backend_cann_buffer_context(int32_t device, void* dev_ptr)
- : device(device),
- dev_ptr(dev_ptr) {}
-
- /**
- * @brief Destructor to free the device memory allocated for the buffer.
- */
- ~ggml_backend_cann_buffer_context() { ACL_CHECK(aclrtFree(dev_ptr)); }
-};
-
-/**
- * @brief Retrieve the name associated with a CANN buffer.
- *
- * This function returns the name of a CANN buffer, which is stored in the
- * context of the buffer.
- *
- * @param buffer The CANN buffer whose name is to be retrieved.
- * @return A pointer to a C-string containing the name of the buffer.
- */
-
-GGML_CALL static const char* ggml_backend_cann_buffer_get_name(
- ggml_backend_buffer_t buffer) {
- return "CANN";
-
- GGML_UNUSED(buffer);
-}
-
-/**
- * @brief Check if a buffer is a CANN buffer.
- *
- * This function checks if a given buffer is a CANN buffer by comparing its
- * `get_name` function pointer to `ggml_backend_cann_buffer_get_name`.
- *
- * @param buffer The buffer to check.
- * @return true if the buffer is a CANN buffer, false otherwise.
- */
-GGML_CALL static bool ggml_backend_buffer_is_cann(
- ggml_backend_buffer_t buffer) {
- return buffer->iface.get_name == ggml_backend_cann_buffer_get_name;
-}
-
-/**
- * @brief Free resources associated with a CANN buffer.
- *
- * This function frees the resources associated with a CANN buffer, including
- * its context.
- *
- * @param buffer The CANN buffer to free.
- */
-GGML_CALL static void ggml_backend_cann_buffer_free_buffer(
- ggml_backend_buffer_t buffer) {
- ggml_backend_cann_buffer_context* ctx =
- (ggml_backend_cann_buffer_context*)buffer->context;
- delete ctx;
-}
-
-/**
- * @brief Retrieve the base pointer of a CANN buffer.
- *
- * This function returns the base pointer of a CANN buffer, which points to the
- * device memory allocated for the buffer.
- *
- * @param buffer The CANN buffer whose base pointer is to be retrieved.
- * @return A pointer to the base of the device memory allocated for the buffer.
- */
-GGML_CALL static void* ggml_backend_cann_buffer_get_base(
- ggml_backend_buffer_t buffer) {
- ggml_backend_cann_buffer_context* ctx =
- (ggml_backend_cann_buffer_context*)buffer->context;
- return ctx->dev_ptr;
-}
-
-/**
- * @brief Transform quantized Q4.0 tensor data into a format suitable for CANN
- * processing.
- *
- * This function transforms quantized Q4.0 tensor data into a format suitable
- * for CANN processing. It extracts quantization values and scales from the
- * source data and prepares them in a format expected by CANN operations.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source data in Q4.0 format.
- * @param dst Pointer to the destination buffer where transformed data will be
- * stored.
- */
-GGML_CALL static void ggml_backend_cann_transform_q4_0(ggml_tensor* tensor,
- const void* src,
- void* dst) {
-
- int64_t n_elems = ggml_nelements(tensor);
- int64_t groups = n_elems / QK4_0;
- size_t quant_bytes = n_elems * sizeof(uint8_t) / 2;
-
- uint8_t* quant_offset = (uint8_t*)dst;
- uint16_t* scale_offset = (uint16_t*)((char*)dst + quant_bytes);
-
- for (int i = 0; i < groups; i++) {
- const block_q4_0* group =
- (const block_q4_0*)((const char*)src + i * sizeof(block_q4_0));
- *scale_offset = group->d;
- scale_offset++;
-
- // 0-15
- for (int j = 0; j < QK4_0 / 2; j += 2) {
- (*quant_offset) = (group->qs[j] & 0x0F);
- (*quant_offset) |= ((group->qs[j + 1] << 4));
- quant_offset++;
- }
-
- // 16-31
- for (int j = 0; j < QK4_0 / 2; j += 2) {
- (*quant_offset) = (group->qs[j] >> 4);
- (*quant_offset) |= (group->qs[j + 1] & 0xF0);
- quant_offset++;
- }
- }
-
- // put (uint4b_t -8) into int4b_t
- for (quant_offset = (uint8_t*)dst;
- quant_offset < (uint8_t*)dst + quant_bytes; quant_offset++) {
- (*quant_offset) ^= 0x88;
- }
-}
-
-/**
- * @brief Transform CANN processed data back into quantized Q4.0 format.
- *
- * This function transforms CANN processed data back into quantized Q4.0 format.
- * It reverses the transformation performed by
- * ggml_backend_cann_transform_q4_0(), converting the data back into its
- * original quantized form.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source buffer containing transformed data.
- * @param dst Pointer to the destination buffer where the Q4.0 formatted data
- * will be stored.
- */
-GGML_CALL static void ggml_backend_cann_transform_back_q4_0(
- const ggml_tensor* tensor, void* src, void* dst) {
-
- int64_t n_elems = ggml_nelements(tensor);
- int64_t groups = n_elems / QK4_0;
- size_t quant_bytes = n_elems * sizeof(uint8_t) / 2;
-
- uint8_t* quant_offset = (uint8_t*)src;
- uint16_t* scale_offset = (uint16_t*)((char*)src + quant_bytes);
-
- for (; quant_offset < (uint8_t*)src + quant_bytes; quant_offset++) {
- (*quant_offset) ^= 0x88;
- }
- quant_offset = (uint8_t*)src;
-
- for (int i = 0; i < groups; i++) {
- block_q4_0* group = (block_q4_0*)((char*)dst + i * sizeof(block_q4_0));
- group->d = *scale_offset;
- scale_offset++;
-
- // 0-15
- for (int j = 0; j < QK4_0 / 2; j += 2) {
- group->qs[j] = ((*quant_offset) & 0x0F);
- group->qs[j + 1] = ((*quant_offset) >> 4);
- quant_offset++;
- }
-
- // 16-31
- for (int j = 0; j < QK4_0 / 2; j += 2) {
- group->qs[j] |= ((*quant_offset) << 4);
- group->qs[j + 1] |= ((*quant_offset) & 0xF0);
- quant_offset++;
- }
- }
-}
-
-/**
- * @brief Transform quantized Q8.0 tensor data into a format suitable for CANN
- * processing.
- *
- * This function transforms quantized Q8.0 tensor data into a format suitable
- * for CANN processing. It extracts quantization values and scales from the
- * source data and prepares them in a format expected by CANN operations.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source data in Q8.0 format.
- * @param dst Pointer to the destination buffer where transformed data will be
- * stored.
- */
-GGML_CALL static void ggml_backend_cann_transform_q8_0(ggml_tensor* tensor,
- const void* src,
- void* dst) {
- int64_t n_elems = ggml_nelements(tensor);
- int64_t groups = n_elems / QK8_0;
- size_t quant_bytes = n_elems * sizeof(uint8_t);
-
- uint8_t* quant_offset = (uint8_t*)dst;
- uint16_t* scale_offset = (uint16_t*)((char*)dst + quant_bytes);
-
- for (int i = 0; i < groups; i++) {
- const block_q8_0* group =
- (const block_q8_0*)((const char*)src + i * sizeof(block_q8_0));
- *scale_offset = group->d;
- scale_offset++;
- size_t group_quant_size = QK8_0 * sizeof(uint8_t);
- memcpy(quant_offset, group->qs, group_quant_size);
- quant_offset += group_quant_size;
- }
-}
-
-/**
- * @brief Transform CANN processed data back into quantized Q8.0 format.
- *
- * This function transforms CANN processed data back into quantized Q8.0 format.
- * It reverses the transformation performed by
- * ggml_backend_cann_transform_q8_0(), converting the data back into its
- * original quantized form.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source buffer containing transformed data.
- * @param dst Pointer to the destination buffer where the Q8.0 formatted data
- * will be stored.
- */
-GGML_CALL static void ggml_backend_cann_transform_back_q8_0(
- const ggml_tensor* tensor, const void* src, void* dst) {
- int64_t n_elems = ggml_nelements(tensor);
- int64_t groups = n_elems / QK8_0;
- size_t quant_bytes = n_elems * sizeof(uint8_t);
-
- const uint8_t* quant_offset = (const uint8_t*)src;
- const uint16_t* scale_offset =
- (const uint16_t*)((const char*)src + quant_bytes);
-
- for (int i = 0; i < groups; i++) {
- block_q8_0* group = (block_q8_0*)((char*)dst + i * sizeof(block_q8_0));
- group->d = *scale_offset;
- scale_offset++;
- size_t group_quant_size = QK8_0 * sizeof(uint8_t);
- memcpy(group->qs, quant_offset, group_quant_size);
- quant_offset += group_quant_size;
- }
-}
-
-/**
- * @brief Transform tensor data based on its type for CANN processing.
- *
- * This function transforms tensor data based on its quantization type for CANN
- * processing. It dispatches the transformation based on the tensor's type to
- * specialized functions handling Q4.0 and Q8.0 formats.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source data to be transformed.
- * @param dst Pointer to the destination buffer where transformed data will be
- * stored.
- */
-GGML_CALL static void ggml_backend_cann_transform(ggml_tensor* tensor,
- const void* src, void* dst) {
- switch (tensor->type) {
- case GGML_TYPE_Q4_0:
- ggml_backend_cann_transform_q4_0(tensor, src, dst);
- break;
- case GGML_TYPE_Q8_0:
- ggml_backend_cann_transform_q8_0(tensor, src, dst);
- break;
- default:
- break;
- }
-}
-
-/**
- * @brief Transform CANN processed data back into tensor data based on its type.
- *
- * This function transforms CANN processed data back into tensor data based on
- * its quantization type for Q4.0 and Q8.0 formats. It dispatches the
- * transformation based on the tensor's type to specialized functions.
- *
- * @param tensor Pointer to the tensor information.
- * @param src Pointer to the source data containing CANN processed data.
- * @param dst Pointer to the destination buffer where transformed tensor data
- * will be stored.
- */
-GGML_CALL static void ggml_backend_cann_transform_back(
- const ggml_tensor* tensor, void* src, void* dst) {
- switch (tensor->type) {
- case GGML_TYPE_Q4_0:
- ggml_backend_cann_transform_back_q4_0(tensor, src, dst);
- break;
- case GGML_TYPE_Q8_0:
- ggml_backend_cann_transform_back_q8_0(tensor, src, dst);
- break;
- default:
- break;
- }
-}
-
-/**
- * @brief Check if transformation is needed for a given tensor type.
- *
- * This function checks if transformation is needed for a given tensor type
- * to prepare data for CANN processing.
- *
- * @param type The tensor type to check.
- * @return true if transformation is needed, false otherwise.
- */
-GGML_CALL static bool need_transform(ggml_type type) {
- switch (type) {
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q8_0:
- return true;
- default:
- return false;
- }
-}
-
-/**
- * @brief Initialize a tensor using data from a CANN buffer.
- *
- * This function initializes a tensor using data from a CANN buffer.
- * It handles special cases such as views and quantization.
- *
- * @param buffer The CANN buffer from which to initialize the tensor.
- * @param tensor Pointer to the tensor to be initialized.
- */
-GGML_CALL static void ggml_backend_cann_buffer_init_tensor(
- ggml_backend_buffer_t buffer, ggml_tensor* tensor) {
- if (tensor->view_src != NULL && tensor->view_offs == 0) {
- GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
- return;
- }
-
- // TODO: can backend doesn't support quantized yet. Just leave the code
- // here.
- if (ggml_is_quantized(tensor->type)) {
- // Initialize padding to 0 to avoid possible NaN values
- size_t original_size = ggml_nbytes(tensor);
- size_t padded_size =
- ggml_backend_buft_get_alloc_size(buffer->buft, tensor);
-
- if (padded_size > original_size && tensor->view_src == nullptr) {
- size_t memset_size = padded_size - original_size;
- ACL_CHECK(aclrtMemset((char*)tensor->data + original_size,
- memset_size, 0, memset_size));
- }
- }
-}
-
-// TODO: need handle tensor which has paddings.
-/**
- * @brief Set tensor data in a CANN buffer.
- *
- * This function sets tensor data in a CANN buffer, handling transformations
- * if needed based on the tensor's type.
- *
- * @param buffer The CANN buffer where the tensor data will be set.
- * @param tensor Pointer to the tensor whose data will be set.
- * @param data Pointer to the source data to be copied into the tensor.
- * @param offset Offset in the source data from where to start copying.
- * @param size Size of the data to be copied, in bytes.
- */
-GGML_CALL static void ggml_backend_cann_buffer_set_tensor(
- ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data,
- size_t offset, size_t size) {
- ggml_backend_cann_buffer_context *ctx =
- (ggml_backend_cann_buffer_context *)buffer->context;
-
- ggml_cann_set_device(ctx->device);
- // TODO: refer to cann(#6017), it use thread's default stream.
- // For acl, synchronous functions use this default stream.
- // Why aclrtSynchronizeDevice?
-
- if (!need_transform(tensor->type)) {
- ACL_CHECK(aclrtMemcpy((char *)tensor->data + offset, size, data, size,
- ACL_MEMCPY_HOST_TO_DEVICE));
- } else {
- void *transform_buffer = malloc(size);
- ggml_backend_cann_transform(tensor, data, transform_buffer);
-
-#ifndef NDEBUG
- void *check_buffer = malloc(size);
- ggml_backend_cann_transform_back(tensor, transform_buffer,
- check_buffer);
- GGML_ASSERT(memcmp(data, check_buffer, size) == 0);
- free(check_buffer);
-#endif
- ACL_CHECK(aclrtMemcpy((char *)tensor->data + offset, size,
- transform_buffer, size,
- ACL_MEMCPY_HOST_TO_DEVICE));
- free(transform_buffer);
- }
-}
-
-/**
- * @brief Get tensor data from a CANN buffer.
- *
- * This function retrieves tensor data from a CANN buffer, handling
- * transformations if needed based on the tensor's type.
- *
- * @param buffer The CANN buffer from which to retrieve tensor data.
- * @param tensor Pointer to the tensor whose data will be retrieved.
- * @param data Pointer to the destination buffer where the tensor data will be
- * copied.
- * @param offset Offset in the destination buffer where to start copying.
- * @param size Size of the data to be copied, in bytes.
- */
-GGML_CALL static void ggml_backend_cann_buffer_get_tensor(
- ggml_backend_buffer_t buffer, const ggml_tensor* tensor, void* data,
- size_t offset, size_t size) {
- ggml_backend_cann_buffer_context* ctx =
- (ggml_backend_cann_buffer_context*)buffer->context;
-
- ggml_cann_set_device(ctx->device);
-
- if (!need_transform(tensor->type)) {
- ACL_CHECK(aclrtMemcpy(data, size, (char*)tensor->data + offset, size,
- ACL_MEMCPY_DEVICE_TO_HOST));
- } else {
- void* transform_buffer = malloc(size);
- ACL_CHECK(aclrtMemcpy(transform_buffer, size,
- (char*)tensor->data + offset, size,
- ACL_MEMCPY_DEVICE_TO_HOST));
- ggml_backend_cann_transform_back(tensor, transform_buffer, data);
- free(transform_buffer);
- }
-}
-
-/**
- * @brief Copy tensor data between CANN buffers if possible.
- *
- * This function copies tensor data between CANN buffers if the source and
- * destination buffers are CANN buffers and they meet the necessary conditions
- * (same device or devices can access each other).
- *
- * @param buffer The destination CANN buffer where the tensor data will be
- * copied.
- * @param src Pointer to the source tensor whose data will be copied.
- * @param dst Pointer to the destination tensor where the data will be copied.
- * @return true if the copy operation succeeded, false otherwise.
- */
-GGML_CALL static bool ggml_backend_cann_buffer_cpy_tensor(
- ggml_backend_buffer_t buffer, const ggml_tensor* src, ggml_tensor* dst) {
- if (ggml_backend_buffer_is_cann(src->buffer)) {
- ggml_backend_cann_buffer_context* src_ctx =
- (ggml_backend_cann_buffer_context*)src->buffer->context;
- ggml_backend_cann_buffer_context* dst_ctx =
- (ggml_backend_cann_buffer_context*)buffer->context;
-
- size_t memcpy_size = ggml_nbytes(src);
- // Same device.
- if (src_ctx->device == dst_ctx->device) {
- ACL_CHECK(aclrtMemcpy((char*)dst->data, memcpy_size,
- (const char*)src->data, memcpy_size,
- ACL_MEMCPY_DEVICE_TO_DEVICE));
- return true;
- } else {
- // Different device but can access by peer.
- int32_t canAccessPeer = 0;
- ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, src_ctx->device,
- dst_ctx->device));
- if (canAccessPeer) {
- ggml_cann_set_device(src_ctx->device);
- ACL_CHECK(aclrtDeviceEnablePeerAccess(dst_ctx->device, 0));
- ACL_CHECK(aclrtMemcpy((char*)dst->data, memcpy_size,
- (const char*)src->data, memcpy_size,
- ACL_MEMCPY_DEVICE_TO_DEVICE));
- return true;
- }
- }
- }
- return false;
-}
-
-/**
- * @brief Clear a CANN buffer by setting all its memory to a specified value.
- *
- * This function clears a CANN buffer by setting all its memory to a specified
- * value.
- *
- * @param buffer The CANN buffer to be cleared.
- * @param value The value to which each byte in the buffer will be set.
- */
-GGML_CALL static void ggml_backend_cann_buffer_clear(
- ggml_backend_buffer_t buffer, uint8_t value) {
- ggml_backend_cann_buffer_context* ctx =
- (ggml_backend_cann_buffer_context*)buffer->context;
-
- ggml_cann_set_device(ctx->device);
- ACL_CHECK(aclrtMemset(ctx->dev_ptr, buffer->size, value, buffer->size));
-}
-
-/**
- * @brief Interface for a CANN buffer in the backend.
- *
- * This structure defines function pointers to operations that can be performed
- * on a CANN buffer within the backend.
- */
-static ggml_backend_buffer_i ggml_backend_cann_buffer_interface = {
- /* .get_name = */ ggml_backend_cann_buffer_get_name,
- /* .free_buffer = */ ggml_backend_cann_buffer_free_buffer,
- /* .get_base = */ ggml_backend_cann_buffer_get_base,
- /* .init_tensor = */ ggml_backend_cann_buffer_init_tensor,
- /* .set_tensor = */ ggml_backend_cann_buffer_set_tensor,
- /* .get_tensor = */ ggml_backend_cann_buffer_get_tensor,
- /* .cpy_tensor = */ ggml_backend_cann_buffer_cpy_tensor,
- /* .clear = */ ggml_backend_cann_buffer_clear,
- /* .reset = */ NULL,
-};
-
-// cann buffer type
-/**
- * @brief Structure representing context information for a specific backend
- * buffer type.
- */
-struct ggml_backend_cann_buffer_type_context {
- int32_t
- device; /**< Device identifier associated with the buffer context. */
- std::string name; /**< Name associated with the buffer context. */
-};
-
-/**
- * @brief Retrieves the name associated with a CANN buffer type.
- *
- * This function returns the descriptive name associated with the specified
- * CANN buffer type context.
- *
- * @param buft Pointer to the buffer type context.
- * @return Const pointer to the C-style string containing the name.
- */
-GGML_CALL static const char* ggml_backend_cann_buffer_type_name(
- ggml_backend_buffer_type_t buft) {
- return "CANN";
-
- GGML_UNUSED(buft);
-}
-
-/**
- * @brief Allocates a new CANN buffer of the specified type and size.
- *
- * This function allocates a new CANN buffer on the specified device with the
- * given size.
- *
- * @param buft Pointer to the buffer type context.
- * @param size Size in bytes of the buffer to allocate.
- * @return Pointer to the allocated buffer, or nullptr if allocation fails.
- */
-GGML_CALL static ggml_backend_buffer_t
-ggml_backend_cann_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft,
- size_t size) {
- ggml_backend_cann_buffer_type_context* buft_ctx =
- (ggml_backend_cann_buffer_type_context*)buft->context;
-
- ggml_cann_set_device(buft_ctx->device);
-
- size = std::max(size, (size_t)1);
-
- void* dev_ptr;
- aclError err = aclrtMalloc(&dev_ptr, size, ACL_MEM_MALLOC_HUGE_FIRST);
- if (err != ACL_SUCCESS) {
- GGML_CANN_LOG_ERROR(
- "%s: allocating %.2f MiB on device %d: aclrtMalloc failed: %s\n",
- __func__, size / 1024.0 / 1024.0, buft_ctx->device,
- aclGetRecentErrMsg());
- return nullptr;
- }
-
- ggml_backend_cann_buffer_context* ctx =
- new ggml_backend_cann_buffer_context(buft_ctx->device, dev_ptr);
-
- return ggml_backend_buffer_init(buft, ggml_backend_cann_buffer_interface,
- ctx, size);
-}
-
-/**
- * @brief Retrieves the memory alignment requirement for CANN buffers of this
- * type.
- *
- * This function returns the alignment requirement in bytes for memory allocated
- * by the CANN buffer type.
- *
- * @param buft Pointer to the buffer type context (unused in this
- * implementation).
- * @return The alignment requirement in bytes (fixed at 128 bytes for CANN
- * buffers).
- */
-GGML_CALL static size_t ggml_backend_cann_buffer_type_get_alignment(
- ggml_backend_buffer_type_t buft) {
- return 128;
-
- GGML_UNUSED(buft);
-}
-
-/**
- * @brief Calculates the allocation size required for a tensor in a CANN buffer.
- *
- * Computes the total allocation size needed for storing the tensor's data in a
- * CANN buffer, considering any necessary padding or adjustments for quantized
- * types.
- *
- * @param buft Pointer to the buffer type context (unused in this
- * implementation).
- * @param tensor Pointer to the tensor for which the allocation size is
- * calculated.
- * @return The total allocation size in bytes required for the tensor in the
- * CANN buffer.
- */
-GGML_CALL static size_t ggml_backend_cann_buffer_type_get_alloc_size(
- ggml_backend_buffer_type_t buft, const ggml_tensor* tensor) {
- size_t size = ggml_nbytes(tensor);
- int64_t ne0 = tensor->ne[0];
-
- // last line must bigger than 32, because every single op deal at
- // least 32 bytes.
- // TODO: quantized type?
- // int64_t line_size = ne0 * ggml_element_size(tensor);
- // int64_t line_size_align_32 = (line_size + 31) & ~31;
- // size += (line_size_align_32 - line_size);
-
- // TODO: not support quantized yet.
- // TODO: consider un-continue tensor.
- if (ggml_is_quantized(tensor->type)) {
- if (ne0 % MATRIX_ROW_PADDING != 0) {
- size += ggml_row_size(
- tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);
- }
- }
-
- return size;
-
- GGML_UNUSED(buft);
-}
-
-/**
- * @brief Interface for managing CANN buffer types in the GGML backend.
- *
- * Provides function pointers for allocating, querying properties, and managing
- * memory for CANN buffer types in the GGML backend.
- */
-static ggml_backend_buffer_type_i ggml_backend_cann_buffer_type_interface = {
- /* .get_name = */ ggml_backend_cann_buffer_type_name,
- /* .alloc_buffer = */ ggml_backend_cann_buffer_type_alloc_buffer,
- /* .get_alignment = */ ggml_backend_cann_buffer_type_get_alignment,
- /* .get_max_size = */ NULL, // defaults to SIZE_MAX
- /* .get_alloc_size = */ ggml_backend_cann_buffer_type_get_alloc_size,
- /* .is_host = */ NULL,
-};
-
-/**
- * @brief Retrieves the CANN buffer type for a specified device.
- *
- * This function initializes and returns the buffer type interface associated
- * with the given device. It ensures thread-safe access using a mutex.
- *
- * @param device The device index for which to retrieve the buffer type.
- * @return A pointer to the buffer type interface for the specified device, or
- * nullptr if the device index is out of range.
- */
-GGML_CALL ggml_backend_buffer_type_t
-ggml_backend_cann_buffer_type(int32_t device) {
- static std::mutex mutex;
- std::lock_guard<std::mutex> lock(mutex);
-
- if (device >= ggml_backend_cann_get_device_count()) {
- return nullptr;
- }
-
- static ggml_backend_buffer_type
- ggml_backend_cann_buffer_types[GGML_CANN_MAX_DEVICES];
-
- static bool ggml_backend_cann_buffer_type_initialized = false;
-
- if (!ggml_backend_cann_buffer_type_initialized) {
- for (int32_t i = 0; i < GGML_CANN_MAX_DEVICES; i++) {
- ggml_backend_cann_buffer_types[i] = {
- /* .iface = */ ggml_backend_cann_buffer_type_interface,
- /* .context = */
- new ggml_backend_cann_buffer_type_context{
- i, "CANN" + std::to_string(i)},
- };
- }
- ggml_backend_cann_buffer_type_initialized = true;
- }
-
- return &ggml_backend_cann_buffer_types[device];
-}
-
-/**
- * @brief Computes the forward operation for a given tensor using CANN
- * operations.
- *
- * This function selects the appropriate CANN operation based on the type of
- * operation specified in the tensor and performs the computation.
- *
- * @param ctx The CANN context containing necessary resources and
- * configurations.
- * @param dst The destination tensor where the result of the computation will be
- * stored.
- * @return true if the computation was successful; false otherwise.
- */
-static bool ggml_cann_compute_forward(ggml_backend_cann_context& ctx,
- struct ggml_tensor* dst) {
- switch (dst->op) {
- case GGML_OP_REPEAT:
- ggml_cann_repeat(ctx, dst);
- break;
- case GGML_OP_GET_ROWS:
- ggml_cann_get_rows(ctx, dst);
- break;
- case GGML_OP_DUP:
- ggml_cann_dup(ctx, dst);
- break;
- case GGML_OP_ADD:
- ggml_cann_add(ctx, dst);
- break;
- case GGML_OP_ACC:
- ggml_cann_acc(ctx, dst);
- break;
- case GGML_OP_MUL:
- ggml_cann_mul_div<aclnnMulGetWorkspaceSize, aclnnMul>(ctx, dst);
- break;
- case GGML_OP_DIV:
- ggml_cann_mul_div<aclnnDivGetWorkspaceSize, aclnnDiv>(ctx, dst);
- break;
- case GGML_OP_UNARY:
- switch (ggml_get_unary_op(dst)) {
- case GGML_UNARY_OP_GELU:
- ggml_cann_activation<aclnnGeluGetWorkspaceSize, aclnnGelu>(
- ctx, dst);
- break;
- case GGML_UNARY_OP_SILU:
- ggml_cann_activation<aclnnSiluGetWorkspaceSize, aclnnSilu>(
- ctx, dst);
- break;
- // TODO: Use faster gelu??
- case GGML_UNARY_OP_GELU_QUICK:
- ggml_cann_activation<aclnnGeluGetWorkspaceSize, aclnnGelu>(
- ctx, dst);
- break;
- case GGML_UNARY_OP_TANH:
- ggml_cann_activation<aclnnTanhGetWorkspaceSize, aclnnTanh>(
- ctx, dst);
- break;
- case GGML_UNARY_OP_RELU:
- ggml_cann_activation<aclnnReluGetWorkspaceSize, aclnnRelu>(
- ctx, dst);
- break;
- case GGML_UNARY_OP_HARDSIGMOID:
- ggml_cann_activation<aclnnHardsigmoidGetWorkspaceSize,
- aclnnHardsigmoid>(ctx, dst);
- break;
- case GGML_UNARY_OP_HARDSWISH:
- ggml_cann_activation<aclnnHardswishGetWorkspaceSize,
- aclnnHardswish>(ctx, dst);
- break;
- default:
- return false;
- }
- break;
- case GGML_OP_NORM:
- ggml_cann_norm(ctx, dst);
- break;
- case GGML_OP_GROUP_NORM:
- ggml_cann_group_norm(ctx, dst);
- break;
- case GGML_OP_CONCAT:
- ggml_cann_concat(ctx, dst);
- break;
- case GGML_OP_UPSCALE:
- ggml_cann_upsample_nearest2d(ctx, dst);
- break;
- case GGML_OP_PAD:
- ggml_cann_pad(ctx, dst);
- break;
- case GGML_OP_ARANGE:
- ggml_cann_arange(ctx, dst);
- break;
- case GGML_OP_TIMESTEP_EMBEDDING:
- ggml_cann_timestep_embedding(ctx, dst);
- break;
- case GGML_OP_LEAKY_RELU:
- ggml_cann_leaky_relu(ctx, dst);
- break;
- case GGML_OP_RMS_NORM:
- ggml_cann_rms_norm(ctx, dst);
- break;
- case GGML_OP_MUL_MAT:
- ggml_cann_mul_mat(ctx, dst);
- break;
- case GGML_OP_MUL_MAT_ID:
- return false;
- case GGML_OP_SCALE:
- ggml_cann_scale(ctx, dst);
- break;
- case GGML_OP_SQR:
- ggml_cann_sqr(ctx, dst);
- break;
- case GGML_OP_CLAMP:
- ggml_cann_clamp(ctx, dst);
- break;
- case GGML_OP_CPY:
- ggml_cann_cpy(ctx, dst);
- break;
- case GGML_OP_CONT:
- ggml_cann_dup(ctx, dst);
- break;
- case GGML_OP_NONE:
- case GGML_OP_RESHAPE:
- case GGML_OP_VIEW:
- case GGML_OP_PERMUTE:
- case GGML_OP_TRANSPOSE:
- break;
- case GGML_OP_DIAG_MASK_INF:
- ggml_cann_diag_mask(ctx, dst, -INFINITY);
- break;
- case GGML_OP_SOFT_MAX:
- ggml_cann_softmax(ctx, dst);
- break;
- case GGML_OP_ROPE:
- ggml_cann_rope(ctx, dst);
- break;
- case GGML_OP_IM2COL:
- ggml_cann_im2col(ctx, dst);
- break;
- case GGML_OP_POOL_2D:
- ggml_cann_pool2d(ctx, dst);
- break;
- case GGML_OP_SUM_ROWS:
- ggml_cann_sum_rows(ctx, dst);
- break;
- case GGML_OP_ARGSORT:
- ggml_cann_argsort(ctx, dst);
- break;
- default:
- return false;
- }
-
- return true;
-}
-
-// backend
-/**
- * @brief Retrieves the name associated with the CANN backend.
- *
- * This function returns the name assigned to the CANN backend, which is stored
- * in the context of the provided backend structure.
- *
- * @param backend Pointer to the CANN backend structure.
- * @return A pointer to a constant string representing the backend name.
- */
-GGML_CALL static const char* ggml_backend_cann_name(ggml_backend_t backend) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- return cann_ctx->name.c_str();
-}
-
-/**
- * @brief Frees resources associated with the CANN backend.
- *
- * This function releases resources associated with the CANN backend context
- * and resets the device associated with the backend to its initial state.
- *
- * @param backend Pointer to the CANN backend structure to be freed.
- */
-GGML_CALL static void ggml_backend_cann_free(ggml_backend_t backend) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
- ACL_CHECK(aclrtSynchronizeDevice());
- ACL_CHECK(aclrtResetDevice(cann_ctx->device));
-
- // finalize when last backend freed.
- if (cann_ctx->device == ggml_backend_cann_get_device_count() - 1) {
- ACL_CHECK(aclFinalize());
- }
-
- delete cann_ctx;
- delete backend;
-}
-
-/**
- * @brief Retrieves the default buffer type associated with the CANN backend.
- *
- * This function returns the buffer type specific to the device associated
- * with the CANN backend. It is used to allocate buffers for computations
- * performed by the backend.
- *
- * @param backend Pointer to the CANN backend structure.
- * @return Pointer to the buffer type structure for the CANN backend.
- */
-GGML_CALL static ggml_backend_buffer_type_t
-ggml_backend_cann_get_default_buffer_type(ggml_backend_t backend) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- return ggml_backend_cann_buffer_type(cann_ctx->device);
-}
-
-/**
- * @brief Sets tensor data asynchronously in the CANN backend.
- *
- * This function asynchronously sets tensor data in the CANN backend. Depending
- * on the tensor type, it may perform data transformations before copying data
- * to the device.
- *
- * @param backend Pointer to the CANN backend structure.
- * @param tensor Pointer to the tensor structure to set data for.
- * @param data Pointer to the host data to copy to the tensor.
- * @param offset Offset in bytes within the host data.
- * @param size Size of the data to copy in bytes.
- */
-GGML_CALL static void ggml_backend_cann_set_tensor_async(ggml_backend_t backend,
- ggml_tensor *tensor,
- const void *data,
- size_t offset,
- size_t size) {
- ggml_backend_cann_context *cann_ctx =
- (ggml_backend_cann_context *)backend->context;
-
- if (!need_transform(tensor->type)) {
- ACL_CHECK(aclrtMemcpyAsync((char *)tensor->data + offset, size, data,
- size, ACL_MEMCPY_HOST_TO_DEVICE,
- cann_ctx->stream()));
- } else {
- void *transform_buffer = malloc(size);
- ggml_backend_cann_transform(tensor, data, transform_buffer);
-
-#ifndef NDEBUG
- void *check_buffer = malloc(size);
- ggml_backend_cann_transform_back(tensor, transform_buffer,
- check_buffer);
- GGML_ASSERT(memcmp(data, check_buffer, size));
- free(check_buffer);
-#endif
- ACL_CHECK(aclrtMemcpyAsync(
- (char *)tensor->data + offset, size, transform_buffer, size,
- ACL_MEMCPY_HOST_TO_DEVICE, cann_ctx->stream()));
- ACL_CHECK(aclrtSynchronizeStream(cann_ctx->stream()));
- free(transform_buffer);
- }
-}
-
-GGML_CALL static void ggml_backend_cann_get_tensor_async(
- ggml_backend_t backend, const ggml_tensor *tensor, void *data,
- size_t offset, size_t size) {
- ggml_backend_cann_context *cann_ctx =
- (ggml_backend_cann_context *)backend->context;
- ggml_backend_buffer_t buf =
- tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-
- GGML_ASSERT(buf->buft == ggml_backend_cann_buffer_type(cann_ctx->device) &&
- "unsupported buffer type");
-
- if (!need_transform(tensor->type)) {
- ACL_CHECK(aclrtMemcpyAsync(data, size, (char *)tensor->data + offset,
- size, ACL_MEMCPY_DEVICE_TO_HOST,
- cann_ctx->stream()));
- } else {
- void *transform_buffer = malloc(size);
- ACL_CHECK(aclrtMemcpyAsync(
- transform_buffer, size, (char *)tensor->data + offset, size,
- ACL_MEMCPY_DEVICE_TO_HOST, cann_ctx->stream()));
- ACL_CHECK(aclrtSynchronizeStream(cann_ctx->stream()));
- ggml_backend_cann_transform_back(tensor, transform_buffer, data);
- free(transform_buffer);
- }
-}
-
-/**
- * @brief Asynchronously copies tensor data between CANN backends.
- *
- * This function copies tensor data asynchronously between two CANN backends. It
- * checks if both tensors reside in CANN buffers and whether the devices support
- * peer-to-peer access for direct copying. If not, it returns false.
- *
- * @param backend_src Pointer to the source CANN backend structure.
- * @param backend_dst Pointer to the destination CANN backend structure.
- * @param src Pointer to the source tensor to copy data from.
- * @param dst Pointer to the destination tensor to copy data to.
- * @return true if the copy operation succeeds, false otherwise.
- */
-GGML_CALL static bool ggml_backend_cann_cpy_tensor_async(
- ggml_backend_t backend_src, ggml_backend_t backend_dst,
- const ggml_tensor* src, ggml_tensor* dst) {
- GGML_ASSERT(ggml_backend_is_cann(backend_src) ||
- ggml_backend_is_cann(backend_dst));
-
- if (!ggml_backend_buffer_is_cann(src->buffer) ||
- !ggml_backend_buffer_is_cann(dst->buffer)) {
- return false;
- }
-
- ggml_backend_buffer_t buf_src =
- src->view_src ? src->view_src->buffer : src->buffer;
- ggml_backend_buffer_t buf_dst =
- dst->view_src ? dst->view_src->buffer : dst->buffer;
-
- ggml_backend_cann_context* cann_ctx_src =
- (ggml_backend_cann_context*)backend_src->context;
- ggml_backend_cann_context* cann_ctx_dst =
- (ggml_backend_cann_context*)backend_dst->context;
-
- size_t copy_size = ggml_nbytes(dst);
- if (backend_src != backend_dst) {
- ggml_backend_cann_buffer_context* buf_ctx_src =
- (ggml_backend_cann_buffer_context*)buf_src->context;
- ggml_backend_cann_buffer_context* buf_ctx_dst =
- (ggml_backend_cann_buffer_context*)buf_dst->context;
-
- GGML_ASSERT(cann_ctx_src->device == buf_ctx_src->device);
- GGML_ASSERT(cann_ctx_dst->device == buf_ctx_dst->device);
-
- int32_t canAccessPeer = 0;
- ACL_CHECK(aclrtDeviceCanAccessPeer(&canAccessPeer, cann_ctx_src->device,
- cann_ctx_dst->device));
- if (!canAccessPeer) {
- return false;
- }
-
- // need open both directions for memcpyasync between devices.
- ggml_cann_set_device(cann_ctx_dst->device);
- ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_src->device, 0));
- ggml_cann_set_device(cann_ctx_src->device);
- ACL_CHECK(aclrtDeviceEnablePeerAccess(cann_ctx_dst->device, 0));
-
- ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size,
- ACL_MEMCPY_DEVICE_TO_DEVICE,
- cann_ctx_src->stream()));
-
- //TODO: workaround for Event didn`t work here.
- aclrtSynchronizeStream(cann_ctx_src->stream());
- } else {
- // src and dst are on the same backend
- ACL_CHECK(aclrtMemcpyAsync(dst->data, copy_size, src->data, copy_size,
- ACL_MEMCPY_DEVICE_TO_DEVICE,
- cann_ctx_dst->stream()));
- }
-
- return true;
-}
-
-/**
- * @brief Synchronizes a CANN backend.
- *
- * This function synchronizes the specified CANN backend by waiting for all
- * operations in its associated stream to complete.
- *
- * @param backend Pointer to the CANN backend structure to synchronize.
- */
-GGML_CALL static void ggml_backend_cann_synchronize(ggml_backend_t backend) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- ggml_cann_set_device(cann_ctx->device);
-
- ACL_CHECK(aclrtSynchronizeStream(cann_ctx->stream()));
-}
-
-/**
- * @brief Computes a computational graph using a CANN backend.
- *
- * This function computes the operations defined in the computational graph
- * using the specified CANN backend.
- *
- * @param backend Pointer to the CANN backend structure to use for computation.
- * @param cgraph Pointer to the computational graph structure containing nodes
- * representing operations to be computed.
- * @return enum ggml_status Returns GGML_STATUS_SUCCESS if computation
- * completes successfully, otherwise an appropriate error status.
- */
-GGML_CALL static enum ggml_status ggml_backend_cann_graph_compute(
- ggml_backend_t backend, ggml_cgraph* cgraph) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- ggml_cann_set_device(cann_ctx->device);
-
- for (int i = 0; i < cgraph->n_nodes; i++) {
- ggml_tensor* node = cgraph->nodes[i];
-
- if (ggml_is_empty(node) || node->op == GGML_OP_NONE) {
- continue;
- }
-
- bool ok = ggml_cann_compute_forward(*cann_ctx, node);
-
- if (!ok) {
- GGML_CANN_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__,
- node->name, ggml_op_name(node->op));
- }
- GGML_ASSERT(ok);
- }
-
- return GGML_STATUS_SUCCESS;
-}
-
-/**
- * @brief Checks if the CANN backend supports a specific operation.
- *
- * This function checks whether the specified operation is supported by the
- * CANN backend.
- *
- * @param backend Pointer to the CANN backend structure to check support for
- * the operation.
- * @param op Pointer to the tensor representing the operation to check.
- * @return bool Returns true if the operation is supported by the backend,
- * otherwise false.
- */
-GGML_CALL static bool ggml_backend_cann_supports_op(ggml_backend_t backend,
- const ggml_tensor* op) {
- switch (op->op) {
- case GGML_OP_UNARY:
- switch (ggml_get_unary_op(op)) {
- case GGML_UNARY_OP_GELU:
- case GGML_UNARY_OP_SILU:
- case GGML_UNARY_OP_RELU:
- case GGML_UNARY_OP_HARDSIGMOID:
- case GGML_UNARY_OP_HARDSWISH:
- case GGML_UNARY_OP_GELU_QUICK:
- case GGML_UNARY_OP_TANH:
- return true;
- default:
- return false;
- }
- case GGML_OP_MUL_MAT: {
- switch (op->src[0]->type) {
- case GGML_TYPE_F16:
- case GGML_TYPE_F32:
- case GGML_TYPE_Q8_0:
- // TODO: fix me
- // Current groupsize should not be greater than k-1 in
- // aclnnWeightQuantBatchMatmulV2GetWorkspaceSize().
- case GGML_TYPE_Q4_0:
- return true;
- default:
- return false;
- }
- }
- case GGML_OP_MUL_MAT_ID:
- return false;
- // embedding
- case GGML_OP_GET_ROWS: {
- switch (op->src[0]->type) {
- case GGML_TYPE_F32:
- case GGML_TYPE_F16:
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q8_0:
- return true;
- default:
- return false;
- }
- } break;
- case GGML_OP_CPY: {
- switch (op->type) {
- case GGML_TYPE_F32:
- case GGML_TYPE_F16:
- case GGML_TYPE_Q8_0:
- case GGML_TYPE_Q4_0:
- return true;
- default:
- return false;
- }
- }
- case GGML_OP_DUP:
- case GGML_OP_REPEAT:
- case GGML_OP_CONCAT:
- case GGML_OP_NONE:
- case GGML_OP_RESHAPE:
- case GGML_OP_VIEW:
- case GGML_OP_PERMUTE:
- case GGML_OP_TRANSPOSE:
- case GGML_OP_NORM:
- case GGML_OP_ADD:
- case GGML_OP_MUL:
- case GGML_OP_DIV:
- case GGML_OP_RMS_NORM:
- case GGML_OP_SCALE:
- case GGML_OP_SQR:
- case GGML_OP_CLAMP:
- case GGML_OP_CONT:
- case GGML_OP_DIAG_MASK_INF:
- case GGML_OP_SOFT_MAX:
- case GGML_OP_ROPE:
- case GGML_OP_IM2COL:
- case GGML_OP_POOL_2D:
- case GGML_OP_SUM_ROWS:
- case GGML_OP_ARGSORT:
- case GGML_OP_ACC:
- case GGML_OP_GROUP_NORM:
- case GGML_OP_UPSCALE:
- case GGML_OP_PAD:
- case GGML_OP_ARANGE:
- case GGML_OP_TIMESTEP_EMBEDDING:
- case GGML_OP_LEAKY_RELU:
- return true;
- default:
- return false;
- }
-
- GGML_UNUSED(backend);
-}
-
-/**
- * @brief Checks if the backend buffer type is associated with the CANN backend.
- *
- * This function checks whether the provided backend buffer type is associated
- * with the CANN backend based on the comparison of its name retrieval function
- * pointer.
- *
- * @param buft Pointer to the backend buffer type to check.
- * @return bool Returns true if the buffer type is associated with the CANN
- * backend, otherwise false.
- */
-static bool ggml_backend_buft_is_cann(ggml_backend_buffer_type_t buft) {
- return buft->iface.get_name == ggml_backend_cann_buffer_type_name;
-}
-
-/**
- * @brief Checks if the CANN backend supports a specific backend buffer type.
- *
- * This function determines whether the CANN backend supports the given backend
- * buffer type by comparing the device context of the backend and buffer type.
- * It returns true if the devices are same between the backend context and
- * buffer type context.
- *
- * @param backend Pointer to the CANN backend.
- * @param buft Pointer to the backend buffer type to check.
- * @return bool Returns true if the CANN backend supports the buffer type,
- * otherwise false.
- */
-GGML_CALL static bool ggml_backend_cann_supports_buft(
- ggml_backend_t backend, ggml_backend_buffer_type_t buft) {
- if (ggml_backend_buft_is_cann(buft)) {
- ggml_backend_cann_context * cann_ctx =
- (ggml_backend_cann_context *)backend->context;
- ggml_backend_cann_buffer_type_context * buft_ctx =
- (ggml_backend_cann_buffer_type_context *)buft->context;
- return buft_ctx->device == cann_ctx->device;
- }
- return false;
-}
-
-/**
- * @brief Determines if a tensor operation should be offloaded to the CANN
- * backend.
- *
- * This function checks if a given tensor operation should be offloaded to the
- * CANN backend based on the operation type and the size of the tensor. It
- * returns true if the second dimension (ne[1]) of the tensor is greater than or
- * equal to the minimum batch size and the operation is not GGML_OP_GET_ROWS.
- *
- * @param backend Pointer to the CANN backend.
- * @param op Pointer to the tensor operation to check.
- * @return bool Returns true if the operation should be offloaded, otherwise
- * false.
- */
-GGML_CALL static bool ggml_backend_cann_offload_op(ggml_backend_t backend,
- const ggml_tensor* op) {
- const int min_batch_size = 32;
- GGML_UNUSED(backend);
-
- return op->ne[1] >= min_batch_size && op->op != GGML_OP_GET_ROWS;
-}
-
-/**
- * @brief Creates a new event for the CANN backend.
- *
- * This function initializes a new event for the CANN backend by setting the
- * device and creating an ACL runtime event. The created event is then wrapped
- * in a ggml_backend_event structure and returned.
- *
- * @param backend Pointer to the CANN backend.
- * @return ggml_backend_event_t Returns a pointer to the new event structure.
- */
-static ggml_backend_event_t ggml_backend_cann_event_new(
- ggml_backend_t backend) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- ggml_cann_set_device(cann_ctx->device);
-
- aclrtEvent event;
- ACL_CHECK(aclrtCreateEvent(&event));
-
- return new ggml_backend_event{
- /* .backend = */ backend,
- /* .context = */ event,
- };
-}
-
-/**
- * @brief Frees a CANN backend event.
- *
- * This function destroys the ACL runtime event associated with the given CANN
- * backend event and then deletes the event structure itself.
- *
- * @param event Pointer to the event structure to be freed.
- */
-static void ggml_backend_cann_event_free(ggml_backend_event_t event) {
- ACL_CHECK(aclrtDestroyEvent((aclrtEvent)event->context));
-
- delete event;
-}
-
-/**
- * @brief Records an event on the CANN backend stream.
- *
- * This function records the given event on the ACL runtime stream associated
- * with the backend context.
- *
- * @param event Pointer to the event structure to be recorded.
- */
-static void ggml_backend_cann_event_record(ggml_backend_event_t event) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)event->backend->context;
-
- ACL_CHECK(aclrtRecordEvent((aclrtEvent)event->context, cann_ctx->stream()));
-}
-
-/**
- * @brief Waits for a recorded event to complete on the CANN backend stream.
- *
- * This function makes the given backend wait for the event to complete on its
- * ACL runtime stream.
- *
- * @param backend Pointer to the backend structure.
- * @param event Pointer to the event structure that the backend needs to wait
- * for.
- */
-static void ggml_backend_cann_event_wait(ggml_backend_t backend,
- ggml_backend_event_t event) {
- ggml_backend_cann_context* cann_ctx =
- (ggml_backend_cann_context*)backend->context;
-
- if (ggml_backend_is_cann(event->backend)) {
- ACL_CHECK(aclrtStreamWaitEvent(cann_ctx->stream(),
- (aclrtEvent)event->context));
- } else {
- GGML_ABORT("fatal error");
- }
-}
-
-/**
- * @brief Synchronizes the given event on the CANN backend.
- *
- * This function waits for the specified event to complete on the ACL runtime.
- *
- * @param event Pointer to the event structure to be synchronized.
- */
-static void ggml_backend_cann_event_synchronize(ggml_backend_event_t event) {
- ACL_CHECK(aclrtSynchronizeEvent((aclrtEvent)event->context));
-}
-
-/**
- * @brief Structure defining the interface for the CANN backend.
- *
- * This structure contains function pointers for various operations
- * supported by the CANN backend, including name retrieval, memory
- * management, tensor operations, synchronization, and event handling.
- */
-static ggml_backend_i ggml_backend_cann_interface = {
- /* .get_name = */ ggml_backend_cann_name,
- /* .free = */ ggml_backend_cann_free,
- /* .get_default_buffer_type = */ ggml_backend_cann_get_default_buffer_type,
- /* .set_tensor_async = */ ggml_backend_cann_set_tensor_async,
- /* .get_tensor_async = */ ggml_backend_cann_get_tensor_async,
- /* .cpy_tensor_async = */ ggml_backend_cann_cpy_tensor_async,
- /* .synchronize = */ ggml_backend_cann_synchronize,
- /* .graph_plan_create = */ NULL,
- /* .graph_plan_free = */ NULL,
- /* .graph_plan_update = */ NULL,
- /* .graph_plan_compute = */ NULL,
- /* .graph_compute = */ ggml_backend_cann_graph_compute,
- /* .supports_op = */ ggml_backend_cann_supports_op,
- /* .supports_buft = */ ggml_backend_cann_supports_buft,
- /* .offload_op = */ ggml_backend_cann_offload_op,
- /* .event_new = */ ggml_backend_cann_event_new,
- /* .event_free = */ ggml_backend_cann_event_free,
- /* .event_record = */ ggml_backend_cann_event_record,
- /* .event_wait = */ ggml_backend_cann_event_wait,
- /* .event_synchronize = */ ggml_backend_cann_event_synchronize,
-};
-
-/**
- * @brief Return the hardcoded GUID for the CANN backend.
- *
- * This function returns a static GUID which uniquely identifies the CANN
- * backend.
- *
- * @return A pointer to the static GUID.
- */
-static ggml_guid_t ggml_backend_cann_guid() {
- static ggml_guid guid = {0xa1, 0x94, 0xaf, 0xac, 0xbd, 0x4f, 0x47, 0x34,
- 0xbe, 0x1a, 0x9e, 0x71, 0x1f, 0x9e, 0xed, 0x64};
- return &guid;
-}
-
-GGML_CALL ggml_backend_t ggml_backend_cann_init(int32_t device) {
- aclInit(nullptr);
- if (device < 0 || device >= ggml_backend_cann_get_device_count()) {
- GGML_CANN_LOG_ERROR("%s: error: invalid device %d\n", __func__, device);
- return nullptr;
- }
-
- ggml_backend_cann_context* ctx = new ggml_backend_cann_context(device);
- if (ctx == nullptr) {
- GGML_CANN_LOG_ERROR("%s: error: failed to allocate context\n", __func__);
- return nullptr;
- }
-
- ggml_backend_t cann_backend =
- new ggml_backend{/* .guid = */ ggml_backend_cann_guid(),
- /* .interface = */ ggml_backend_cann_interface,
- /* .context = */ ctx};
-
- return cann_backend;
-}
-
-GGML_CALL bool ggml_backend_is_cann(ggml_backend_t backend) {
- return backend != NULL &&
- ggml_guid_matches(backend->guid, ggml_backend_cann_guid());
-}
-
-GGML_CALL int32_t ggml_backend_cann_get_device_count() {
- return ggml_cann_info().device_count;
-}
-
-GGML_CALL void ggml_backend_cann_get_device_description(
- int32_t device, char* description, size_t description_size) {
- ggml_cann_set_device(device);
- const char* soc_name = aclrtGetSocName();
- snprintf(description, description_size, "%s", soc_name);
-}
-
-GGML_CALL void ggml_backend_cann_get_device_memory(int32_t device, size_t* free,
- size_t* total) {
- ggml_cann_set_device(device);
- ACL_CHECK(aclrtGetMemInfo(ACL_HBM_MEM, free, total));
-}
-
-// backend registry
-/**
- * @brief Initializes a CANN backend based on the provided parameters.
- *
- * This function initializes a CANN backend using the device index and then
- * initializes the backend using `ggml_backend_cann_init`.
- *
- * @param params Parameters for initialization (unused in this implementation).
- * @param user_data User data containing the device index to initialize the
- * backend.
- * @return ggml_backend_t The initialized CANN backend.
- */
-GGML_CALL static ggml_backend_t ggml_backend_reg_cann_init(const char* params,
- void* user_data) {
- ggml_backend_t cann_backend =
- ggml_backend_cann_init((int)(intptr_t)user_data);
- return cann_backend;
-
- GGML_UNUSED(params);
-}
-
-extern "C" GGML_CALL int ggml_backend_cann_reg_devices();
-
-/**
- * @brief Registers CANN (Ascend) devices as backend options.
- *
- * This function initializes ACL, retrieves the number of available CANN
- * devices, and registers each device as a backend option using
- * `ggml_backend_register`. Each device is given a unique name based on
- * `GGML_CANN_NAME` followed by its index.
- *
- * @return int The number of CANN devices registered.
- */
-GGML_CALL int ggml_backend_cann_reg_devices() {
- uint32_t device_count = ggml_backend_cann_get_device_count();
- // initialization
- for (uint32_t i = 0; i < device_count; i++) {
- char name[128];
- snprintf(name, sizeof(name), "CANN%d", i);
- ggml_backend_register(name, ggml_backend_reg_cann_init,
- ggml_backend_cann_buffer_type(i),
- (void*)(intptr_t)i);
- }
- return device_count;
-}
overview / pkg / ggml / sources / ggml / commitdiff