cl_program program_mul_mv_q4_0_f32_1d_16x_flat;
cl_program program_mul_mv_q6_K;
cl_program program_mul_mv_mxfp4_f32;
+ cl_program program_mul_mv_mxfp4_f32_flat;
cl_program program_mul_mv_f16_f16;
cl_program program_mul_mv_f16_f32_1row;
cl_program program_mul_mv_f16_f32_l4;
cl_program program_tsembd;
cl_program program_mul_mv_id_q4_0_f32_8x_flat;
cl_program program_mul_mv_id_mxfp4_f32;
+ cl_program program_mul_mv_id_mxfp4_f32_flat;
cl_program program_mul_mm_f32_f32_l4_lm;
cl_program program_mul_mm_f16_f32_l4_lm;
cl_kernel kernel_mul_mat_f16_f32_tiled;
cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v;
cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0;
+ cl_kernel kernel_convert_block_mxfp4, kernel_restore_block_mxfp4;
cl_kernel kernel_mul_mat_q4_0_f32_8x_flat;
cl_kernel kernel_convert_block_q4_0_noshuffle;
cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat;
cl_kernel kernel_mul_mv_q6_K_f32;
- cl_kernel kernel_mul_mv_mxfp4_f32;
+ cl_kernel kernel_mul_mv_mxfp4_f32, kernel_mul_mv_mxfp4_f32_flat;
cl_kernel kernel_im2col_f32, kernel_im2col_f16;
cl_kernel kernel_argsort_f32_i32;
cl_kernel kernel_sum_rows_f32;
cl_kernel kernel_timestep_embedding;
cl_kernel kernel_mul_mv_id_q4_0_f32_8x_flat;
cl_kernel kernel_mul_mv_id_mxfp4_f32;
+ cl_kernel kernel_mul_mv_id_mxfp4_f32_flat;
cl_kernel kernel_mul_mm_f32_f32_l4_lm;
cl_kernel kernel_mul_mm_f16_f32_l4_lm;
CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_noshuffle", &err), err));
CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0", &err), err));
CL_CHECK((backend_ctx->kernel_restore_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0", &err), err));
+ CL_CHECK((backend_ctx->kernel_convert_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_mxfp4", &err), err));
+ CL_CHECK((backend_ctx->kernel_restore_block_mxfp4 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_mxfp4", &err), err));
GGML_LOG_CONT(".");
}
GGML_LOG_CONT(".");
}
+ // mul_mv_mxfp4_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_mxfp4_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_mxfp4_f32_flat.cl");
+#endif
+ backend_ctx->program_mul_mv_mxfp4_f32_flat =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_mul_mv_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_mxfp4_f32_flat, "kernel_mul_mv_mxfp4_f32_flat", &err), err));
+ GGML_LOG_CONT(".");
+ }
+
// mul_mv_f16_f16
{
#ifdef GGML_OPENCL_EMBED_KERNELS
GGML_LOG_CONT(".");
}
+ // mul_mv_id_mxfp4_f32_flat
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "mul_mv_id_mxfp4_f32_flat.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("mul_mv_id_mxfp4_f32_flat.cl");
+#endif
+ backend_ctx->program_mul_mv_id_mxfp4_f32_flat =
+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);
+
+ CL_CHECK((backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat = clCreateKernel(backend_ctx->program_mul_mv_id_mxfp4_f32_flat, "kernel_mul_mv_id_mxfp4_f32_flat", &err), err));
+ GGML_LOG_CONT(".");
+ }
+
// Adreno kernels
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
// transpose
}
};
+struct ggml_tensor_extra_cl_mxfp4 {
+ // Quantized values.
+ cl_mem q = nullptr;
+ // Quantized values in image1d_buffer_t.
+ cl_mem q_img = nullptr;
+ // Scales in E8M0.
+ cl_mem e = nullptr;
+ // Scales in image1d_buffer_t.
+ cl_mem e_img = nullptr;
+ // Size of quantized values.
+ size_t size_q = 0;
+ // Size of scales.
+ size_t size_e = 0;
+
+ ~ggml_tensor_extra_cl_mxfp4() {
+ reset();
+ }
+
+ void reset() {
+ // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.
+ // They must be properly released so that the original buffer can be
+ // properly released to avoid memory leak.
+ if (q != nullptr) {
+ CL_CHECK(clReleaseMemObject(q));
+ q = nullptr;
+ }
+ if (e != nullptr) {
+ CL_CHECK(clReleaseMemObject(e));
+ e = nullptr;
+ }
+ if (q != nullptr) {
+ CL_CHECK(clReleaseMemObject(q_img));
+ q = nullptr;
+ }
+ // Currently, q_img and d_img are only initialized when SMALL_ALLOC is
+ // enabled. They point to the images in ggml_backend_opencl_buffer_context.
+ // So, there is no need to release them here.
+ // TODO: initialize them for non SMALL_PATH path, or remove them.
+ q_img = nullptr;
+ e_img = nullptr;
+ size_q = 0;
+ size_e = 0;
+ }
+};
+
//------------------------------------------------------------------------------
// Backend API
//------------------------------------------------------------------------------
for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) {
delete e;
}
+ for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4) {
+ delete e;
+ }
+ for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) {
+ delete e;
+ }
}
ggml_tensor_extra_cl * ggml_opencl_alloc_temp_tensor_extra() {
return extra;
}
+ ggml_tensor_extra_cl_mxfp4 * ggml_opencl_alloc_temp_tensor_extra_mxfp4() {
+ ggml_tensor_extra_cl_mxfp4 * extra;
+ if (temp_tensor_extras_mxfp4.empty()) {
+ extra = new ggml_tensor_extra_cl_mxfp4();
+ } else {
+ extra = temp_tensor_extras_mxfp4.back();
+ temp_tensor_extras_mxfp4.pop_back();
+ }
+
+ temp_tensor_extras_mxfp4_in_use.push_back(extra);
+
+ extra->reset();
+ return extra;
+ }
+
void reset() {
for (ggml_tensor_extra_cl * e : temp_tensor_extras_in_use) {
temp_tensor_extras.push_back(e);
temp_tensor_extras_q4_0.push_back(e);
}
temp_tensor_extras_q4_0_in_use.clear();
+
+ for (ggml_tensor_extra_cl_mxfp4 * e : temp_tensor_extras_mxfp4_in_use) {
+ temp_tensor_extras_mxfp4.push_back(e);
+ }
+ temp_tensor_extras_mxfp4_in_use.clear();
}
// Pools for extras. Available extras are in `temp_tensor_extras`. Extras
std::vector<ggml_tensor_extra_cl *> temp_tensor_extras_in_use;
std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0;
std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0_in_use;
+ std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4;
+ std::vector<ggml_tensor_extra_cl_mxfp4 *> temp_tensor_extras_mxfp4_in_use;
// The buffer_context is initially created by ggml_backend_buft_alloc_buffer
// before any tensor is initialized (at the beginning of alloc_tensor_range).
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
+ return;
+
+ }
+ if (tensor->type == GGML_TYPE_MXFP4) {
+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;
+ GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");
+
+ // Allocate the new extra and create aliases from the original.
+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;
+ ggml_tensor_extra_cl_mxfp4 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_mxfp4();
+
+ size_t size_e = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(char);
+ size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*ggml_blck_size(tensor->type)/2;
+ GGML_ASSERT(size_e + size_q == ggml_nbytes(tensor) && "Incorrect tensor size");
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+ CL_CHECK(clEnqueueWriteBuffer(
+ queue, data_device, CL_TRUE, 0,
+ ggml_nbytes(tensor), data, 0, NULL, NULL));
+
+ // The original tensor memory is divided into scales and quants, i.e.,
+ // we first store scales, then quants.
+ cl_buffer_region region;
+
+ // Create subbuffer for scales.
+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);
+ region.size = size_e;
+ extra->e = clCreateSubBuffer(
+ extra_orig->data_device, CL_MEM_READ_WRITE,
+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+ auto previous_origin = region.origin;
+
+ // Create subbuffer for quants.
+ region.origin = align_to(previous_origin + size_e, backend_ctx->alignment);
+ region.size = size_q;
+ extra->q = clCreateSubBuffer(
+ extra_orig->data_device, CL_MEM_READ_WRITE,
+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
+ CL_CHECK(err);
+
+ cl_kernel kernel = backend_ctx->kernel_convert_block_mxfp4;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->e));
+
+ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
+ size_t local_work_size[] = {64, 1, 1};
+
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clReleaseMemObject(data_device));
+
+ // Create image for Q
+ cl_image_format img_format_q = {CL_RG, CL_UNSIGNED_INT32};
+ cl_image_desc img_desc_q = {
+ CL_MEM_OBJECT_IMAGE1D_BUFFER,
+ static_cast<size_t>(ggml_nelements(tensor)/32*2),
+ 0, 0, 0, 0, 0, 0, 0,
+ { extra->q }
+ };
+ extra->q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_format_q, &img_desc_q, NULL, &err);
+
+ tensor->extra = extra;
+
return;
}
#endif // GGML_OPENCL_SOA_Q
size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
size_t local_work_size[] = {1, 1, 1};
+ cl_event evt;
+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,
+ global_work_size, local_work_size, 0, NULL, &evt));
+ CL_CHECK(clWaitForEvents(1, &evt));
+ CL_CHECK(clEnqueueReadBuffer(
+ queue, data_device, CL_TRUE, offset,
+ size, data, 0, NULL, NULL));
+ CL_CHECK(clReleaseMemObject(data_device));
+ return;
+ } else if (tensor->type == GGML_TYPE_MXFP4) {
+ ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *)tensor->extra;
+
+ cl_int err;
+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,
+ ggml_nbytes(tensor), NULL, &err);
+ CL_CHECK(err);
+
+ cl_kernel kernel = backend_ctx->kernel_restore_block_mxfp4;
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->e));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));
+
+ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};
+ size_t local_work_size[] = {1, 1, 1};
+
cl_event evt;
CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL,
global_work_size, local_work_size, 0, NULL, &evt));
CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL));
CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_d, buf_d, 0, NULL, NULL));
CL_CHECK(clFinish(queue));
+ } else if (tensor->type == GGML_TYPE_MXFP4) {
+ ggml_tensor_extra_cl_mxfp4 * extra = (ggml_tensor_extra_cl_mxfp4 *) tensor->extra;
+ GGML_ASSERT(extra);
+
+ size_t size_q = ggml_nelements(tensor)/QK_MXFP4 * QK_MXFP4/2;
+ size_t size_e = ggml_nelements(tensor)/QK_MXFP4 * sizeof(char);
+ GGML_ASSERT(size_q + size_e == ggml_nbytes(tensor));
+ buf_q = malloc(size_q);
+ buf_d = malloc(size_e);
+
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->q, CL_TRUE, 0, size_q, buf_q, 0, NULL, NULL));
+ CL_CHECK(clEnqueueReadBuffer(queue, extra->d, CL_TRUE, 0, size_e, buf_d, 0, NULL, NULL));
+ CL_CHECK(clFinish(queue));
} else {
// Read out the tensor from GPU memory.
ggml_tensor_extra_cl * extra = (ggml_tensor_extra_cl *) tensor->extra;
#ifdef GGML_OPENCL_SOA_Q
ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;
+ ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra;
#endif
const int ne00 = src0 ? src0->ne[0] : 0;
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));
break;
case GGML_TYPE_MXFP4: {
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_mxfp4_f32_flat;
+
+ cl_mem q;
+ if (backend_ctx->gpu_family == INTEL) {
+ nth0 = 16;
+ nth1 = 2;
+ ndst = nth1*2;
+
+ q = extra0_mxfp4->q;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ nth0 = 64;
+ nth1 = 2;
+ ndst = nth1*2;
+
+ q = extra0_mxfp4->q_img;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3));
+#else
kernel = backend_ctx->kernel_mul_mv_mxfp4_f32;
if (backend_ctx->gpu_family == INTEL) {
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r3));
CL_CHECK(clSetKernelArg(kernel, 18, sizeof(float)*nth0,nullptr));
+#endif
break;
}
default:
cl_ulong offset2 = extra2->offset + src2->view_offs;
cl_ulong offsetd = extrad->offset + dst->view_offs;
+ GGML_UNUSED(offset0);
+
#ifdef GGML_OPENCL_SOA_Q
ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;
+ ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)src0->extra;
#endif
const int ne00 = src0->ne[0];
break;
}
case GGML_TYPE_MXFP4: {
+#ifdef GGML_OPENCL_SOA_Q
+ kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32_flat;
+
+ cl_mem q;
+ if (backend_ctx->gpu_family == INTEL) {
+ sgs = 16;
+ nsg = 2;
+ ndst = 2;
+
+ q = extra0_mxfp4->q;
+ } else if (backend_ctx->gpu_family == ADRENO) {
+ sgs = 64;
+ nsg = 1;
+ ndst = 4;
+
+ q = extra0_mxfp4->q_img;
+ } else {
+ GGML_ASSERT(false && "TODO: Unknown GPU");
+ }
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_mxfp4->e));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra2->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offset2));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11));
+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12));
+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb11));
+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb12));
+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb13));
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &ne20));
+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne21));
+ CL_CHECK(clSetKernelArg(kernel, 19, sizeof(cl_ulong), &nb21));
+ CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &ne0));
+ CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &ne1));
+ CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));
+ CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));
+#else // GGML_OPENCL_SOA_Q
kernel = backend_ctx->kernel_mul_mv_id_mxfp4_f32;
if (backend_ctx->gpu_family == INTEL) {
CL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));
CL_CHECK(clSetKernelArg(kernel, 24, sizeof(float)*sgs,nullptr));
-
+#endif // GGML_OPENCL_SOA_Q
break;
}
default:
--- /dev/null
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_subgroups
+#pragma OPENCL EXTENSION cl_intel_subgroups : enable
+#else
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#endif
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_MXFP4 32
+
+static inline half4 mxfp4_to_fp16_packed(ushort fp4x4) {
+ ushort2 fp16_packed_a, fp16_packed_b, bias_a, bias_b, sign_a, sign_b;
+ fp16_packed_a.lo = (fp4x4 << 9) & 0x0E00;
+ fp16_packed_a.hi = (fp4x4 << 5) & 0x0E00;
+ fp16_packed_b.lo = (fp4x4 << 1) & 0x0E00;
+ fp16_packed_b.hi = (fp4x4 >> 3) & 0x0E00;
+
+ bias_a.lo = (fp16_packed_a.lo == 0) ? 0x0 : 0x3800;
+ bias_a.hi = (fp16_packed_a.hi == 0) ? 0x0 : 0x3800;
+ bias_b.lo = (fp16_packed_b.lo == 0) ? 0x0 : 0x3800;
+ bias_b.hi = (fp16_packed_b.hi == 0) ? 0x0 : 0x3800;
+
+ fp16_packed_a.lo = (fp16_packed_a.lo == 0x0200) ? 0x0 : fp16_packed_a.lo;
+ fp16_packed_a.hi = (fp16_packed_a.hi == 0x0200) ? 0x0 : fp16_packed_a.hi;
+ fp16_packed_b.lo = (fp16_packed_b.lo == 0x0200) ? 0x0 : fp16_packed_b.lo;
+ fp16_packed_b.hi = (fp16_packed_b.hi == 0x0200) ? 0x0 : fp16_packed_b.hi;
+
+ sign_a.lo = (fp4x4 << 12) & 0x8000;
+ sign_a.hi = (fp4x4 << 8) & 0x8000;
+ sign_b.lo = (fp4x4 << 4) & 0x8000;
+ sign_b.hi = fp4x4 & 0x8000;
+
+ fp16_packed_a = sign_a + bias_a + fp16_packed_a;
+ fp16_packed_b = sign_b + bias_b + fp16_packed_b;
+
+ return as_half4((ushort4)(fp16_packed_a, fp16_packed_b));
+}
+
+static inline float e8m0_to_fp32(uchar x) {
+ int bits;
+ bits = (x == 0) ? 0x00400000 : ((uint) x << 23);
+ return as_float(bits);
+}
+
+#ifdef INTEL_GPU
+#define N_R0_MXFP4 2 // number of rows each subgroup works on
+#define N_SG_MXFP4 2 // number of subgroups in a work group
+#define N_SIMDWIDTH 16 // subgroup size
+#elif defined (ADRENO_GPU)
+#define N_R0_MXFP4 4
+#define N_SG_MXFP4 1
+#define N_SIMDWIDTH 64
+#define SRC0Q_IMG
+#endif
+
+kernel void kernel_mul_mv_id_mxfp4_f32_flat(
+#ifdef SRC0Q_IMG
+ __read_only image1d_buffer_t src0_q,
+#else
+ global uchar * src0_q,
+#endif
+ global uchar * src0_e,
+ global uchar * src1,
+ ulong offset1,
+ global uchar * src2,
+ ulong offset2,
+ global uchar * dst,
+ ulong offsetd,
+ int ne00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne11,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne20,
+ int ne21,
+ ulong nb21,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ dst = dst + offsetd;
+
+ const int iid1 = get_group_id(2) / ne20;
+ const int idx = get_group_id(2) % ne20;
+
+ uint i02 = ((global uint *) (src2 + offset2 + iid1 * nb21))[idx];
+
+ int i11 = idx % ne11;
+
+ int nb = ne00 / QK_MXFP4;
+
+ uint src0_off = i02*nb02;
+ src0_off /= 17; // 17 = sizeof(block_mxfp4)
+
+ src0_e = src0_e + src0_off;
+
+ dst = dst + (idx * ne0 + iid1 * ne1 * ne0) * sizeof(float);
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+
+ int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;
+
+ uint offset_src0 = first_row*nb01;
+ offset_src0 /= 17; // 17 = sizeof(block_mxfp4)
+#ifdef SRC0Q_IMG
+ ulong offset_q = src0_off + offset_src0;
+#else
+ src0_q = src0_q + src0_off*16;
+ global uchar16 * x_q = (global uchar16 *)(src0_q) + offset_src0;
+#endif
+ global uchar * x_e = src0_e + offset_src0;
+
+ const short ix = get_sub_group_local_id() >> 1;
+ const short it = get_sub_group_local_id() & 1;
+
+ float sumf[N_R0_MXFP4] = {0.f};
+
+ src1 = src1 + offset1 + i11 * nb11 + iid1 * nb12;
+ global float * y = (global float *) (src1 + r1 * nb11);
+ global float * yb = y + ix * QK_MXFP4 + it * 8;
+
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH / 2) {
+ global float4 * y4 = (global float4 *)yb;
+
+ #pragma unroll
+ for (short row = 0; row < N_R0_MXFP4; row++) {
+ uchar xb_e = x_e[row * nb + ib];
+#ifdef SRC0Q_IMG
+ ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy);
+#else
+ ushort4 xb_q = vload4(0, (global ushort *)((global uchar *)(x_q + row * nb + ib) + 8 * it));
+#endif
+
+ half4 fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s0);
+ half4 fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s1);
+ float4 acc1 = y4[0] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
+ acc1 += y4[4] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
+
+ fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s2);
+ fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s3);
+ acc1 += y4[1] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
+ acc1 += y4[5] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
+
+ sumf[row] += e8m0_to_fp32(xb_e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));
+ }
+
+ yb += (N_SIMDWIDTH / 2) * QK_MXFP4;
+ }
+
+ global float * dst_f32 = (global float *)dst + (ulong)r1 * ne0;
+
+ for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {
+ float sum_all = sub_group_reduce_add(sumf[row]);
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = sum_all;
+ }
+ }
+}
--- /dev/null
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+#ifdef cl_intel_subgroups
+#pragma OPENCL EXTENSION cl_intel_subgroups : enable
+#else
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
+#endif
+
+#ifdef cl_intel_required_subgroup_size
+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
+#define INTEL_GPU 1
+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
+#elif defined(cl_qcom_reqd_sub_group_size)
+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
+#define ADRENO_GPU 1
+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))
+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
+#endif
+
+#define QK_MXFP4 32
+
+static inline half4 mxfp4_to_fp16_packed(ushort fp4x4) {
+ ushort2 fp16_packed_a, fp16_packed_b, bias_a, bias_b, sign_a, sign_b;
+ fp16_packed_a.lo = (fp4x4 << 9) & 0x0E00;
+ fp16_packed_a.hi = (fp4x4 << 5) & 0x0E00;
+ fp16_packed_b.lo = (fp4x4 << 1) & 0x0E00;
+ fp16_packed_b.hi = (fp4x4 >> 3) & 0x0E00;
+
+ bias_a.lo = (fp16_packed_a.lo == 0) ? 0x0 : 0x3800;
+ bias_a.hi = (fp16_packed_a.hi == 0) ? 0x0 : 0x3800;
+ bias_b.lo = (fp16_packed_b.lo == 0) ? 0x0 : 0x3800;
+ bias_b.hi = (fp16_packed_b.hi == 0) ? 0x0 : 0x3800;
+
+ fp16_packed_a.lo = (fp16_packed_a.lo == 0x0200) ? 0x0 : fp16_packed_a.lo;
+ fp16_packed_a.hi = (fp16_packed_a.hi == 0x0200) ? 0x0 : fp16_packed_a.hi;
+ fp16_packed_b.lo = (fp16_packed_b.lo == 0x0200) ? 0x0 : fp16_packed_b.lo;
+ fp16_packed_b.hi = (fp16_packed_b.hi == 0x0200) ? 0x0 : fp16_packed_b.hi;
+
+ sign_a.lo = (fp4x4 << 12) & 0x8000;
+ sign_a.hi = (fp4x4 << 8) & 0x8000;
+ sign_b.lo = (fp4x4 << 4) & 0x8000;
+ sign_b.hi = fp4x4 & 0x8000;
+
+ fp16_packed_a = sign_a + bias_a + fp16_packed_a;
+ fp16_packed_b = sign_b + bias_b + fp16_packed_b;
+
+ return as_half4((ushort4)(fp16_packed_a, fp16_packed_b));
+}
+
+static inline float e8m0_to_fp32(uchar x) {
+ int bits;
+ bits = (x == 0) ? 0x00400000 : ((uint) x << 23);
+ return as_float(bits);
+}
+
+#ifdef INTEL_GPU
+#define N_R0_MXFP4 2 // number of rows each subgroup works on
+#define N_SG_MXFP4 2 // number of subgroups in a work group
+#define N_SIMDWIDTH 16 // subgroup size
+#elif defined (ADRENO_GPU)
+#define N_R0_MXFP4 2
+#define N_SG_MXFP4 2
+#define N_SIMDWIDTH 64
+#define SRC0Q_IMG
+#endif
+
+#ifdef INTEL_GPU
+REQD_SUBGROUP_SIZE_16
+#elif defined (ADRENO_GPU)
+REQD_SUBGROUP_SIZE_64
+#endif
+kernel void kernel_mul_mv_mxfp4_f32_flat(
+#ifdef SRC0Q_IMG
+ __read_only image1d_buffer_t src0_q,
+#else
+ global uchar * src0_q,
+#endif
+ global uchar * src0_e,
+ global uchar * src1,
+ ulong offset1,
+ global uchar * dst,
+ ulong offsetd,
+ int ne00,
+ ulong nb01,
+ ulong nb02,
+ ulong nb03,
+ int ne12,
+ ulong nb11,
+ ulong nb12,
+ ulong nb13,
+ int ne0,
+ int ne1,
+ int r2,
+ int r3
+) {
+ src1 = src1 + offset1;
+ dst = dst + offsetd;
+
+ int nb = ne00 / QK_MXFP4;
+
+ int r0 = get_group_id(0);
+ int r1 = get_group_id(1);
+ int im = get_group_id(2);
+
+ int first_row = (r0 * N_SG_MXFP4 + get_sub_group_id()) * N_R0_MXFP4;
+
+ uint i12 = im % ne12;
+ uint i13 = im / ne12;
+
+ uint offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;
+ // 17 = sizeof(block_mxfp4)
+ offset_src0 /= 17;
+#ifdef SRC0Q_IMG
+ ulong offset_q = offset_src0;
+#else
+ global uchar16 * x_q = (global uchar16 *)(src0_q) + offset_src0;
+#endif
+ global uchar * x_e = src0_e + offset_src0;
+
+ ulong offset_src1 = r1 * nb11 + i12 * nb12 + i13 * nb13;
+ global float * y = (global float *)(src1 + offset_src1);
+
+ const short ix = get_sub_group_local_id() >> 1; // 0...15
+ const short it = get_sub_group_local_id() & 1; // 0 or 1
+
+ float sumf[N_R0_MXFP4] = {0.f};
+
+ global float * yb = y + ix * QK_MXFP4 + it * 8;
+
+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
+ global float4 * y4 = (global float4 *)yb;
+
+ #pragma unroll
+ for (short row = 0; row < N_R0_MXFP4; row++) {
+ uchar xb_e = x_e[row * nb + ib];
+#ifdef SRC0Q_IMG
+ ushort4 xb_q = as_ushort4(read_imageui(src0_q, (offset_q + row * nb + ib) * 2 + it).xy);
+#else
+ ushort4 xb_q = vload4(0, (global ushort *)((global uchar *)(x_q + row * nb + ib) + 8 * it));
+#endif
+
+ half4 fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s0);
+ half4 fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s1);
+ float4 acc1 = y4[0] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
+ acc1 += y4[4] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
+
+ fp16x4_0 = mxfp4_to_fp16_packed(xb_q.s2);
+ fp16x4_1 = mxfp4_to_fp16_packed(xb_q.s3);
+ acc1 += y4[1] * (float4)(fp16x4_0.s0, fp16x4_0.s2, fp16x4_1.s0, fp16x4_1.s2);
+ acc1 += y4[5] * (float4)(fp16x4_0.s1, fp16x4_0.s3, fp16x4_1.s1, fp16x4_1.s3);
+
+ sumf[row] += e8m0_to_fp32(xb_e) * ((acc1.s0 + acc1.s1) + (acc1.s2 + acc1.s3));
+ }
+
+ yb += (N_SIMDWIDTH/2) * QK_MXFP4;
+ }
+
+ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;
+
+ for (int row = 0; row < N_R0_MXFP4 && first_row + row < ne0; ++row) {
+ float sum_all = sub_group_reduce_add(sumf[row]);
+ if (get_sub_group_local_id() == 0) {
+ dst_f32[first_row + row] = sum_all;
+ }
+ }
+}
overview / pkg / ggml / sources / llama.cpp / commitdiff