GGML_ASSERT(ggml_are_same_shape(src0, dst));
- GGML_ASSERT(src0->nb[0] == sizeof(float));
-
const int ith = params->ith;
const int nth = params->nth;
for (int64_t i03 = 0; i03 < ne03; i03++) {
for (int64_t i02 = 0; i02 < ne02; i02++) {
for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
- const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
- float sum = 0.0;
- ggml_vec_sum_f32(ne00, &sum, x);
- float mean = sum/ne00;
+ if (nb00 == sizeof(float) && nb0 == sizeof(float)) {
+ const float * xf = (const float *) x;
- float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
- float variance = 0;
+ float sum = 0.0;
+ ggml_vec_sum_f32(ne00, &sum, xf);
+ float mean = sum/ne00;
+
+ float * yf = (float *) y;
+ float variance = 0;
#ifdef GGML_USE_ACCELERATE
- mean = -mean;
- vDSP_vsadd(x, 1, &mean, y, 1, ne00);
- vDSP_measqv(y, 1, &variance, ne00);
+ mean = -mean;
+ vDSP_vsadd(xf, 1, &mean, yf, 1, ne00);
+ vDSP_measqv(yf, 1, &variance, ne00);
#else
- variance = ggml_vec_cvar_f32(ne00, y, x, mean);
+ variance = ggml_vec_cvar_f32(ne00, yf, xf, mean);
#endif //GGML_USE_ACCELERATE
- const float scale = 1.0f/sqrtf(variance + eps);
- ggml_vec_scale_f32(ne00, y, scale);
+ const float scale = 1.0f/sqrtf(variance + eps);
+ ggml_vec_scale_f32(ne00, yf, scale);
+ } else {
+ float sum = 0.0;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sum += *(const float *) (x + i00*nb00);
+ }
+ const float mean = sum/ne00;
+
+ float variance = 0.0f;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const float v = *(const float *) (x + i00*nb00) - mean;
+ *(float *) (y + i00*nb0) = v;
+ variance += v * v;
+ }
+ variance /= ne00;
+
+ const float scale = 1.0f/sqrtf(variance + eps);
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ *(float *) (y + i00*nb0) *= scale;
+ }
+ }
}
}
}
GGML_ASSERT(ggml_are_same_shape(src0, dst));
- GGML_ASSERT(src0->nb[0] == sizeof(float));
-
const int ith = params->ith;
const int nth = params->nth;
for (int64_t i03 = 0; i03 < ne03; i03++) {
for (int64_t i02 = 0; i02 < ne02; i02++) {
for (int64_t i01 = ith; i01 < ne01; i01 += nth) {
- const float * x = (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03);
+ const char * x = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
ggml_float sum = 0.0;
for (int64_t i00 = 0; i00 < ne00; i00++) {
- sum += (ggml_float)(x[i00] * x[i00]);
+ const float xi = *(const float *) (x + i00*nb00);
+ sum += (ggml_float)(xi * xi);
}
- float * y = (float *) ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3);
-
- memcpy(y, x, ne00 * sizeof(float));
-
const float scale = 1.0f/fmaxf(sqrtf(sum), eps);
- ggml_vec_scale_f32(ne00, y, scale);
+ char * y = (char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
+
+ if (nb00 == sizeof(float) && nb0 == sizeof(float)) {
+ memcpy(y, x, ne00 * sizeof(float));
+ ggml_vec_scale_f32(ne00, (float *) y, scale);
+ } else {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const float xi = *(const float *) (x + i00*nb00);
+ *(float *) (y + i00*nb0) = xi * scale;
+ }
+ }
}
}
}
case GGML_OP_NORM:
case GGML_OP_RMS_NORM:
case GGML_OP_L2_NORM:
- return true;
+ return ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_RMS_NORM_BACK:
return ggml_is_contiguous(op->src[0]);
break;
vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
vk_pipeline pipeline_scale_f32;
- vk_pipeline pipeline_sqr_f32;
- vk_pipeline pipeline_sqrt_f32;
- vk_pipeline pipeline_sin_f32;
- vk_pipeline pipeline_cos_f32;
vk_pipeline pipeline_log[2];
vk_pipeline pipeline_tri[2];
vk_pipeline pipeline_diag[2];
- vk_pipeline pipeline_clamp_f32;
+ vk_pipeline pipeline_clamp[2];
vk_pipeline pipeline_pad_f32;
vk_pipeline pipeline_roll_f32;
vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;
vk_pipeline pipeline_gelu_quick[2];
vk_pipeline pipeline_silu[2];
vk_pipeline pipeline_relu[2];
+ vk_pipeline pipeline_sqr[2];
+ vk_pipeline pipeline_sqrt[2];
+ vk_pipeline pipeline_sin[2];
+ vk_pipeline pipeline_cos[2];
vk_pipeline pipeline_xielu[2];
vk_pipeline pipeline_neg[2];
vk_pipeline pipeline_tanh[2];
vk_pipeline pipeline_geglu_erf[2];
vk_pipeline pipeline_geglu_quick[2];
- vk_pipeline pipeline_leaky_relu_f32;
+ vk_pipeline pipeline_leaky_relu[2];
vk_pipeline pipeline_silu_back_f32;
vk_pipeline pipeline_diag_mask_inf_f32;
vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;
}
ggml_vk_create_pipeline(device, device->pipeline_mul_mat_vec_nc_f16_f32, "mul_mat_vec_nc_f16_f32", mul_mat_vec_nc_f16_f32_len, mul_mat_vec_nc_f16_f32_data, "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_nc_push_constants), {1, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
+ ggml_vk_create_pipeline(device, device->pipeline_norm_f32, "norm_f32", norm_f32_len, norm_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {1, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_group_norm_f32, "group_norm_f32", group_norm_f32_len, group_norm_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_f32, "rms_norm_f32", rms_norm_f32_len, rms_norm_f32_data, "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_sqr_f32, "sqr_f32", sqr_f32_len, sqr_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_sqrt_f32, "sqrt_f32", sqrt_f32_len, sqrt_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_sin_f32, "sin_f32", sin_f32_len, sin_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_cos_f32, "cos_f32", cos_f32_len, cos_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_clamp_f32, "clamp_f32", clamp_f32_len, clamp_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
CREATE_UNARY(gelu_quick)
CREATE_UNARY(silu)
CREATE_UNARY(relu)
+ CREATE_UNARY(sqr)
+ CREATE_UNARY(sqrt)
+ CREATE_UNARY(sin)
+ CREATE_UNARY(cos)
+ CREATE_UNARY(clamp)
+ CREATE_UNARY(leaky_relu)
CREATE_UNARY(xielu)
CREATE_UNARY(neg)
CREATE_UNARY(tanh)
CREATE_GLU(geglu_quick)
#undef CREATE_GLU
- ggml_vk_create_pipeline(device, device->pipeline_leaky_relu_f32, "leaky_relu_f32", leaky_relu_f32_len, leaky_relu_f32_data, "main", 2, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_diag_mask_inf_f32, "diag_mask_inf_f32", diag_mask_inf_f32_len, diag_mask_inf_f32_data, "main", 2, sizeof(vk_op_diag_mask_push_constants), {1, 512, 1}, {}, 1, true);
}
return nullptr;
case GGML_OP_SQR:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_sqr_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_SQRT:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_sqrt_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_SIN:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_sin_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_COS:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_cos_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_LOG:
}
return nullptr;
case GGML_OP_CLAMP:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_clamp_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_PAD:
}
return nullptr;
case GGML_OP_LEAKY_RELU:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_leaky_relu_f32;
+ if (src0->type == dst->type &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
+ return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16];
}
return nullptr;
case GGML_OP_CONV_2D:
case GGML_OP_TRI:
case GGML_OP_DIAG:
case GGML_OP_CLAMP:
+ case GGML_OP_LEAKY_RELU:
case GGML_OP_PAD:
case GGML_OP_ROLL:
case GGML_OP_REPEAT:
static void ggml_vk_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
float * op_params = (float *)dst->op_params;
+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
+ p.param1 = op_params[0];
- ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, { (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], op_params[0], 0.0f, 0.0f, 0.0f });
+ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_NORM, std::move(p));
}
static void ggml_vk_group_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
static void ggml_vk_leaky_relu(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
const float * op_params = (const float *)dst->op_params;
- ggml_vk_op_f32<vk_op_push_constants>(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, { (uint32_t)ggml_nelements(src0), 0, op_params[0], 0.0f, 0.0f, 0.0f });
+ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst);
+ p.param1 = op_params[0];
+
+ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_LEAKY_RELU, std::move(p));
}
#ifdef GGML_VULKAN_RUN_TESTS
case GGML_OP_TRANSPOSE:
case GGML_OP_RMS_NORM:
return true;
- case GGML_OP_NORM:
case GGML_OP_GROUP_NORM:
return ggml_is_contiguous(op->src[0]);
+ case GGML_OP_NORM:
case GGML_OP_L2_NORM:
- return ggml_is_contiguous_rows(op->src[0]) &&
- op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
+ return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
case GGML_OP_ADD:
case GGML_OP_SUB:
case GGML_OP_MUL:
case GGML_OP_SIN:
case GGML_OP_COS:
case GGML_OP_CLAMP:
- return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_LEAKY_RELU:
+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
+ op->type == op->src[0]->type;
case GGML_OP_OPT_STEP_ADAMW:
case GGML_OP_OPT_STEP_SGD:
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
+++ /dev/null
-#version 450
-
-#include "types.glsl"
-#include "generic_unary_head.glsl"
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-void main() {
- const uint idx = get_idx();
-
- if (idx >= p.ne) {
- return;
- }
-
- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val < p.param1 ? p.param1 : (val > p.param2 ? p.param2 : val));
-}
+++ /dev/null
-#version 450
-
-#include "types.glsl"
-#include "generic_unary_head.glsl"
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-void main() {
- const uint idx = get_idx();
-
- if (idx >= p.ne) {
- return;
- }
-
- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(cos(val));
-}
const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
const uint tid = gl_LocalInvocationID.x;
- const uint i3 = row / (p.ne11 * p.ne12);
- const uint i3_offset = i3 * p.ne12 * p.ne11;
- const uint i2 = (row - i3_offset) / p.ne11;
- const uint i2_offset = i2 * p.ne11;
- const uint i1 = row - i3_offset - i2_offset;
+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);
sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
- const FLOAT_TYPE xi = FLOAT_TYPE(data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0]);
+ const FLOAT_TYPE xi = FLOAT_TYPE(data_a[a_base + i0*p.nb00]);
sum[tid] += xi * xi;
}
const FLOAT_TYPE scale = 1.0f / max(sqrt(sum[0]), FLOAT_TYPE(p.param1));
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
- data_d[i3*p.nb13 + i2*p.nb12 + i1*p.nb11 + i0] = D_TYPE(scale * FLOAT_TYPE(data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0]));
+ data_d[d_base + i0*p.nb10] = D_TYPE(scale * FLOAT_TYPE(data_a[a_base + i0*p.nb00]));
}
}
+++ /dev/null
-#version 450
-
-#include "generic_head.glsl"
-#include "types.glsl"
-
-#extension GL_EXT_control_flow_attributes : enable
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
-layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
-
-void main() {
- const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
-
- if (i >= p.KX) {
- return;
- }
-
- const float val = float(data_a[i]);
- data_d[i] = D_TYPE(max(val, 0.0f) + min(val, 0.0f) * p.param1);
-}
#version 450
-#include "generic_head.glsl"
#include "types.glsl"
+#include "generic_unary_head.glsl"
#extension GL_EXT_control_flow_attributes : enable
#define BLOCK_SIZE 512
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
-layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
-layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
-
shared vec2 sum[BLOCK_SIZE];
void main() {
const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
const uint tid = gl_LocalInvocationID.x;
+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);
+
sum[tid] = vec2(0.0f, 0.0f);
- [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
- const float xi = float(data_a[row*p.KX + col]);
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
+ const float xi = float(data_a[a_base + i0*p.nb00]);
sum[tid].x += xi;
sum[tid].y += xi * xi;
}
barrier();
}
- const float mean = sum[0].x / p.KX;
- const float var = sum[0].y / p.KX - mean * mean;
+ const float mean = sum[0].x / p.ne00;
+ const float var = sum[0].y / p.ne00 - mean * mean;
const float inv_std = inversesqrt(var + p.param1);
- [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
- data_d[row*p.KX + col] = D_TYPE((float(data_a[row*p.KX + col]) - mean) * inv_std);
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
+ data_d[d_base + i0*p.nb10] = D_TYPE((float(data_a[a_base + i0*p.nb00]) - mean) * inv_std);
}
}
+++ /dev/null
-#version 450
-
-#include "types.glsl"
-#include "generic_unary_head.glsl"
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-void main() {
- const uint idx = get_idx();
-
- if (idx >= p.ne) {
- return;
- }
-
- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(sin(val));
-}
+++ /dev/null
-#version 450
-
-#include "types.glsl"
-#include "generic_unary_head.glsl"
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-void main() {
- const uint idx = get_idx();
-
- if (idx >= p.ne) {
- return;
- }
-
- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(sqrt(val));
-}
+++ /dev/null
-#version 450
-
-#include "types.glsl"
-#include "generic_unary_head.glsl"
-
-layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
-
-void main() {
- const uint idx = get_idx();
-
- if (idx >= p.ne) {
- return;
- }
-
- const FLOAT_TYPE val = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val * val);
-}
return -x;
}
+float op_sqr(float x) {
+ return x * x;
+}
+
+float op_sqrt(float x) {
+ return sqrt(x);
+}
+
+float op_sin(float x) {
+ return sin(x);
+}
+
+float op_cos(float x) {
+ return cos(x);
+}
+
+float op_clamp(float x) {
+ return clamp(x, p.param1, p.param2);
+}
+
+float op_leaky_relu(float x) {
+ return max(x, 0.0f) + min(x, 0.0f) * p.param1;
+}
+
float op_step(float x) {
return x >= 0.0f ? 1.0f : 0.0f;
}
string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
- string_to_spv("sqr_f32", "square.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
- string_to_spv("sqrt_f32", "sqrt.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
- string_to_spv("sin_f32", "sin.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
- string_to_spv("cos_f32", "cos.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
- string_to_spv("clamp_f32", "clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("concat_i8", "concat.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}});
string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}});
string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}});
string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}});
+ string_to_spv("sqr_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqr"}});
+ string_to_spv("sqr_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqr"}});
+ string_to_spv("sqrt_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqrt"}});
+ string_to_spv("sqrt_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqrt"}});
+ string_to_spv("sin_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sin"}});
+ string_to_spv("sin_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sin"}});
+ string_to_spv("cos_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_cos"}});
+ string_to_spv("cos_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_cos"}});
+ string_to_spv("clamp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_clamp"}});
+ string_to_spv("clamp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_clamp"}});
+ string_to_spv("leaky_relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_leaky_relu"}});
+ string_to_spv("leaky_relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_leaky_relu"}});
string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}});
string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}});
string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}});
string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
string_to_spv("geglu_quick_f32","geglu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("leaky_relu_f32", "leaky_relu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("diag_mask_inf_f32", "diag_mask_inf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
case GGML_OP_RMS_NORM:
case GGML_OP_NORM:
case GGML_OP_L2_NORM:
- supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32;
+ supports_op = (op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32) && ggml_is_contiguous_rows(src0);
break;
case GGML_OP_ROPE:
supports_op = op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
const std::array<int64_t, 4> ne;
const bool v; // whether a is a non-contiguous view
const float eps;
+ const bool noncontig_rows;
std::string vars() override {
- return VARS_TO_STR4(type, ne, v, eps);
+ return VARS_TO_STR5(type, ne, v, eps, noncontig_rows);
}
test_norm(ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {64, 5, 4, 3},
bool v = false,
- float eps = 1e-6f)
- : type(type), ne(ne), v(v), eps(eps) {}
+ float eps = 1e-6f,
+ bool noncontig_rows = false)
+ : type(type), ne(ne), v(v), eps(eps), noncontig_rows(noncontig_rows) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
- ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
+ const std::array<int64_t, 4> ne_a = noncontig_rows ?
+ std::array<int64_t, 4>{ ne[1], ne[0], ne[2], ne[3] } : ne;
+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());
ggml_set_name(a, "a");
+ if (noncontig_rows) {
+ a = ggml_permute(ctx, a, 1, 0, 2, 3);
+ ggml_set_name(a, "permuted a");
+ }
if (v) {
a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "view of a");
const std::array<int64_t, 4> ne;
const float eps;
bool v;
+ bool noncontig_rows;
std::string vars() override {
- return VARS_TO_STR4(type, ne, eps, v);
+ return VARS_TO_STR5(type, ne, eps, v, noncontig_rows);
}
test_l2_norm(ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {64, 64, 320, 1},
float eps = 1e-12f,
- bool v = false)
- : type(type), ne(ne), eps(eps), v(v) {}
+ bool v = false,
+ bool noncontig_rows = false)
+ : type(type), ne(ne), eps(eps), v(v), noncontig_rows(noncontig_rows) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
- ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
+ const std::array<int64_t, 4> ne_a = noncontig_rows ?
+ std::array<int64_t, 4>{ ne[1], ne[0], ne[2], ne[3] } : ne;
+ ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne_a.data());
ggml_set_name(a, "a");
+ if (noncontig_rows) {
+ a = ggml_permute(ctx, a, 1, 0, 2, 3);
+ ggml_set_name(a, "permuted a");
+ }
if (v) {
a = ggml_view_4d(ctx, a, a->ne[0]/2, a->ne[1]/2, a->ne[2]/2, a->ne[3]/2, a->nb[1], a->nb[2], a->nb[3], 0);
ggml_set_name(a, "view of a");
test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));
test_cases.emplace_back(new test_rms_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, v, eps));
}
+ test_cases.emplace_back(new test_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, false, eps, true));
test_cases.emplace_back(new test_rms_norm_back(GGML_TYPE_F32, { n, 5, 4, 3 }, eps));
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false));
test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, true));
+ test_cases.emplace_back(new test_l2_norm(GGML_TYPE_F32, { n, 5, 4, 3 }, eps, false, true));
}
}
overview / pkg / ggml / sources / llama.cpp / commitdiff