From: sjinzh Date: Sun, 2 Jul 2023 16:36:53 +0000 (+0800) Subject: zig : add tests codes using zig (#315) X-Git-Tag: upstream/0.0.1642~1360 X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=497d8e586cb0ab226075f4a32d9b2404694627a6;p=pkg%2Fggml%2Fsources%2Fggml zig : add tests codes using zig (#315) * update build.zig * zig : add tests by zig * zig : add tests codes using zig * zig : add tests codes using zig --- diff --git a/build.zig b/build.zig index 87151a88..a4f9016e 100644 --- a/build.zig +++ b/build.zig @@ -1,27 +1,29 @@ const std = @import("std"); -// Zig Version: 0.11.0-dev.3798+a5e15eced +// Zig Version: 0.11.0-dev.3886+0c1bfe271 // Zig Build Command: zig build -// Zig Run Command: zig build -h -// zig build run_dolly-v2 -// zig build run_gpt-2 -// zig build run_gpt-j -// zig build run_gpt-neox -// zig build run_mnist -// zig build run_mpt -// zig build run_replit -// zig build run_starcoder -// zig build run_test-grad0 -// zig build run_test-mul-mat0 -// zig build run_test-mul-mat2 -// zig build run_test-opt -// zig build run_test-vec1 -// zig build run_test0 -// zig build run_test1 -// zig build run_test2 +// Zig Run Command: zig build -h +// zig build run_dolly-v2 +// zig build run_gpt-2 +// zig build run_gpt-j +// zig build run_gpt-neox +// zig build run_mnist +// zig build run_mpt +// zig build run_replit +// zig build run_starcoder +// zig build run_test-grad0 +// zig build run_test-mul-mat0 +// zig build run_test-mul-mat2 +// zig build run_test-opt +// zig build run_test-vec1 +// zig build run_test0 +// zig build run_test1 +// zig build run_test2 // zig build run_test3 // zig build run_zig_test0 -// zig build run_zig_test1 +// zig build run_zig_test1 +// zig build run_zig_test2 +// zig build run_zig_test3 pub fn build(b: *std.build.Builder) void { const target = b.standardTargetOptions(.{}); const optimize = b.standardOptimizeOption(.{}); @@ -117,6 +119,8 @@ pub fn build(b: *std.build.Builder) void { const zig_tests = .{ "test0", "test1", + "test2", + "test3", }; inline for (zig_tests) |name| { const exe = b.addExecutable(.{ @@ -135,4 +139,4 @@ pub fn build(b: *std.build.Builder) void { const run_step = b.step("run_zig_" ++ name, "Run zig_tests"); run_step.dependOn(&run_cmd.step); } -} \ No newline at end of file +} diff --git a/tests/test0.zig b/tests/test0.zig index 8246a978..e47bf365 100644 --- a/tests/test0.zig +++ b/tests/test0.zig @@ -1,7 +1,5 @@ const std = @import("std"); const c = @cImport({ - @cInclude("stdio.h"); - @cInclude("stdlib.h"); @cInclude("ggml/ggml.h"); }); diff --git a/tests/test1.zig b/tests/test1.zig index e40c37e8..60fed53d 100644 --- a/tests/test1.zig +++ b/tests/test1.zig @@ -1,7 +1,5 @@ const std = @import("std"); const c = @cImport({ - @cInclude("stdio.h"); - @cInclude("stdlib.h"); @cInclude("ggml/ggml.h"); }); @@ -114,7 +112,7 @@ pub fn main() !void { c.ggml_graph_dump_dot(&gf, null, "test1-2-forward.dot"); c.ggml_graph_dump_dot(&gb, &gf, "test1-2-backward.dot"); } - + /////////////////////////////////////////////////////////////// { @@ -182,7 +180,7 @@ pub fn main() !void { try std.testing.expect(c.ggml_get_f32_1d(y, 0) == 12.0); try std.testing.expect(c.ggml_get_f32_1d(x1.*.grad, 0) == 24.0); try std.testing.expect(c.ggml_get_f32_1d(x2.*.grad, 0) == 12.0); - try std.testing.expect(c.ggml_get_f32_1d(x3.*.grad, 0) == 4.0); + try std.testing.expect(c.ggml_get_f32_1d(x3.*.grad, 0) == 4.0); const g1 = x1.*.grad; const g2 = x2.*.grad; @@ -197,16 +195,16 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gbb)); - std.debug.print("H * [1, 1, 1] = [ {d:.6} {d:.6} {d:.6}]\n", - .{ - c.ggml_get_f32_1d(x1.*.grad, 0), + std.debug.print("H * [1, 1, 1] = [ {d:.6} {d:.6} {d:.6}]\n", + .{ + c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x3.*.grad, 0), }); try std.testing.expect(c.ggml_get_f32_1d(x1.*.grad, 0) == 56.0); try std.testing.expect(c.ggml_get_f32_1d(x2.*.grad, 0) == 34.0); - try std.testing.expect(c.ggml_get_f32_1d(x3.*.grad, 0) == 12.0); + try std.testing.expect(c.ggml_get_f32_1d(x3.*.grad, 0) == 12.0); c.ggml_graph_dump_dot(&gf, null, "test1-4-forward.dot"); c.ggml_graph_dump_dot(&gb, &gf, "test1-4-backward.dot"); @@ -235,13 +233,13 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gb)); std.debug.print("y = {d:.6}\n", .{c.ggml_get_f32_1d(y, 0)}); - std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x1.*.grad, 1), c.ggml_get_f32_1d(x1.*.grad, 2), }); - std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 1), @@ -292,13 +290,13 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gb)); std.debug.print("y = {d:.6}\n", .{c.ggml_get_f32_1d(y, 0)}); - std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x1.*.grad, 1), c.ggml_get_f32_1d(x1.*.grad, 2), }); - std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 1), @@ -349,13 +347,13 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gb)); std.debug.print("y = {d:.6}\n", .{c.ggml_get_f32_1d(y, 0)}); - std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x1.*.grad, 1), c.ggml_get_f32_1d(x1.*.grad, 2), }); - std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 1), @@ -400,13 +398,13 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gb)); std.debug.print("y = {d:.6}\n", .{c.ggml_get_f32_1d(y, 0)}); - std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x1.*.grad, 1), c.ggml_get_f32_1d(x1.*.grad, 2), }); - std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 1), @@ -430,13 +428,13 @@ pub fn main() !void { c.ggml_graph_compute(ctx0, @constCast(&gb)); std.debug.print("y = {d:.6}\n", .{c.ggml_get_f32_1d(y, 0)}); - std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx1 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x1.*.grad, 0), c.ggml_get_f32_1d(x1.*.grad, 1), c.ggml_get_f32_1d(x1.*.grad, 2), }); - std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", + std.debug.print("df/dx2 = {d:.6} {d:.6} {d:.6}\n", .{ c.ggml_get_f32_1d(x2.*.grad, 0), c.ggml_get_f32_1d(x2.*.grad, 1), @@ -456,4 +454,4 @@ pub fn main() !void { } _ = try std.io.getStdIn().reader().readByte(); -} \ No newline at end of file +} diff --git a/tests/test2.zig b/tests/test2.zig new file mode 100644 index 00000000..667de967 --- /dev/null +++ b/tests/test2.zig @@ -0,0 +1,165 @@ +const std = @import("std"); +const Thread = std.Thread; +const c = @cImport({ + @cInclude("ggml/ggml.h"); +}); + +fn is_close(a: f32, b: f32, epsilon: f32) bool { + return std.math.fabs(a - b) < epsilon; +} + +pub fn main() !void { + const params = .{ + .mem_size = 128*1024*1024, + .mem_buffer = null, + .no_alloc = false, + }; + + var opt_params = c.ggml_opt_default_params(c.GGML_OPT_LBFGS); + + const nthreads = try Thread.getCpuCount(); + opt_params.n_threads = @intCast(nthreads); + std.debug.print("test2: n_threads:{}\n", .{opt_params.n_threads}); + + const xi = [_]f32{ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0 }; + const yi = [_]f32{ 15.0, 25.0, 35.0, 45.0, 55.0, 65.0, 75.0, 85.0, 95.0, 105.0 }; + + const n = xi.len; + + const ctx0 = c.ggml_init(params); + defer c.ggml_free(ctx0); + + const x = c.ggml_new_tensor_1d(ctx0, c.GGML_TYPE_F32, n); + const y = c.ggml_new_tensor_1d(ctx0, c.GGML_TYPE_F32, n); + + for (0..n) |i| { + const x_data_pointer: [*]f32 = @ptrCast(@alignCast(x.*.data)); + x_data_pointer[i] = xi[i]; + const y_data_pointer: [*]f32 = @ptrCast(@alignCast(y.*.data)); + y_data_pointer[i] = yi[i]; + } + + { + const t0 = c.ggml_new_f32(ctx0, 0.0); + const t1 = c.ggml_new_f32(ctx0, 0.0); + + // initialize auto-diff parameters: + _ = c.ggml_set_param(ctx0, t0); + _ = c.ggml_set_param(ctx0, t1); + + // f = sum_i[(t0 + t1*x_i - y_i)^2]/(2n) + const f = + c.ggml_div(ctx0, + c.ggml_sum(ctx0, + c.ggml_sqr(ctx0, + c.ggml_sub(ctx0, + c.ggml_add(ctx0, + c.ggml_mul(ctx0, x, c.ggml_repeat(ctx0, t1, x)), + c.ggml_repeat(ctx0, t0, x)), + y) + ) + ), + c.ggml_new_f32(ctx0, @as(f32, 2.0)*n)); + + const res = c.ggml_opt(null, opt_params, f); + + std.debug.print("t0 = {d:.6}\n", .{c.ggml_get_f32_1d(t0, 0)}); + std.debug.print("t1 = {d:.6}\n", .{c.ggml_get_f32_1d(t1, 0)}); + + try std.testing.expect(res == c.GGML_OPT_OK); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t0, 0), 5.0, 1e-3)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t1, 0), 10.0, 1e-3)); + } + + { + const t0 = c.ggml_new_f32(ctx0, -1.0); + const t1 = c.ggml_new_f32(ctx0, 9.0); + + _ = c.ggml_set_param(ctx0, t0); + _ = c.ggml_set_param(ctx0, t1); + + // f = 0.5*sum_i[abs(t0 + t1*x_i - y_i)]/n + const f = + c.ggml_mul(ctx0, + c.ggml_new_f32(ctx0, @as(f32, 1.0)/(2*n)), + c.ggml_sum(ctx0, + c.ggml_abs(ctx0, + c.ggml_sub(ctx0, + c.ggml_add(ctx0, + c.ggml_mul(ctx0, x, c.ggml_repeat(ctx0, t1, x)), + c.ggml_repeat(ctx0, t0, x)), + y) + ) + ) + ); + + + const res = c.ggml_opt(null, opt_params, f); + + try std.testing.expect(res == c.GGML_OPT_OK); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t0, 0), 5.0, 1e-2)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t1, 0), 10.0, 1e-2)); + } + + { + const t0 = c.ggml_new_f32(ctx0, 5.0); + const t1 = c.ggml_new_f32(ctx0, -4.0); + + _ = c.ggml_set_param(ctx0, t0); + _ = c.ggml_set_param(ctx0, t1); + + // f = t0^2 + t1^2 + const f = + c.ggml_add(ctx0, + c.ggml_sqr(ctx0, t0), + c.ggml_sqr(ctx0, t1) + ); + + const res = c.ggml_opt(null, opt_params, f); + + try std.testing.expect(res == c.GGML_OPT_OK); + try std.testing.expect(is_close(c.ggml_get_f32_1d(f, 0), 0.0, 1e-3)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t0, 0), 0.0, 1e-3)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t1, 0), 0.0, 1e-3)); + } + + ///////////////////////////////////////// + + { + const t0 = c.ggml_new_f32(ctx0, -7.0); + const t1 = c.ggml_new_f32(ctx0, 8.0); + + _ = c.ggml_set_param(ctx0, t0); + _ = c.ggml_set_param(ctx0, t1); + + // f = (t0 + 2*t1 - 7)^2 + (2*t0 + t1 - 5)^2 + const f = + c.ggml_add(ctx0, + c.ggml_sqr(ctx0, + c.ggml_sub(ctx0, + c.ggml_add(ctx0, + t0, + c.ggml_mul(ctx0, t1, c.ggml_new_f32(ctx0, 2.0))), + c.ggml_new_f32(ctx0, 7.0) + ) + ), + c.ggml_sqr(ctx0, + c.ggml_sub(ctx0, + c.ggml_add(ctx0, + c.ggml_mul(ctx0, t0, c.ggml_new_f32(ctx0, 2.0)), + t1), + c.ggml_new_f32(ctx0, 5.0) + ) + ) + ); + + const res = c.ggml_opt(null, opt_params, f); + + try std.testing.expect(res == c.GGML_OPT_OK); + try std.testing.expect(is_close(c.ggml_get_f32_1d(f, 0), 0.0, 1e-3)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t0, 0), 1.0, 1e-3)); + try std.testing.expect(is_close(c.ggml_get_f32_1d(t1, 0), 3.0, 1e-3)); + } + + _ = try std.io.getStdIn().reader().readByte(); +} diff --git a/tests/test3.zig b/tests/test3.zig new file mode 100644 index 00000000..d676961f --- /dev/null +++ b/tests/test3.zig @@ -0,0 +1,102 @@ +const std = @import("std"); +const Thread = std.Thread; +const c = @cImport({ + @cInclude("stdlib.h"); + @cInclude("ggml/ggml.h"); +}); + +fn is_close(a: f32, b: f32, epsilon: f32) bool { + return std.math.fabs(a - b) < epsilon; +} + +pub fn main() !void { + const params = .{ + .mem_size = 128*1024*1024, + .mem_buffer = null, + .no_alloc = false, + }; + + var opt_params = c.ggml_opt_default_params(c.GGML_OPT_LBFGS); + + const nthreads = try Thread.getCpuCount(); + opt_params.n_threads = @intCast(nthreads); + + const NP = 1 << 12; + const NF = 1 << 8; + + const ctx0 = c.ggml_init(params); + defer c.ggml_free(ctx0); + + const F = c.ggml_new_tensor_2d(ctx0, c.GGML_TYPE_F32, NF, NP); + const l = c.ggml_new_tensor_1d(ctx0, c.GGML_TYPE_F32, NP); + + // regularization weight + const lambda = c.ggml_new_f32(ctx0, 1e-5); + + c.srand(0); + + const l_data_pointer: [*]f32 = @ptrCast(@alignCast(l.*.data)); + const f_data_pointer: [*]f32 = @ptrCast(@alignCast(F.*.data)); + for (0..NP) |j| { + const ll = if (j < NP/2) @as(f32, 1.0) else @as(f32, -1.0); + l_data_pointer[j] = ll; + + for (0..NF) |i| { + const c_rand: f32 = @floatFromInt(c.rand()); + f_data_pointer[j*NF + i] = + ((if (ll > 0 and i < NF/2) @as(f32, 1.0) else + if (ll < 0 and i >= NF/2) @as(f32, 1.0) else @as(f32, 0.0)) + + (c_rand/c.RAND_MAX - 0.5) * 0.1) / (0.5 * NF); + } + } + + { + // initial guess + const x = c.ggml_set_f32(c.ggml_new_tensor_1d(ctx0, c.GGML_TYPE_F32, NF), 0.0); + + c.ggml_set_param(ctx0, x); + + // f = sum[(fj*x - l)^2]/n + lambda*|x^2| + const f = + c.ggml_add(ctx0, + c.ggml_div(ctx0, + c.ggml_sum(ctx0, + c.ggml_sqr(ctx0, + c.ggml_sub(ctx0, + c.ggml_mul_mat(ctx0, F, x), + l) + ) + ), + c.ggml_new_f32(ctx0, @as(f32, NP)) + ), + c.ggml_mul(ctx0, + c.ggml_sum(ctx0, c.ggml_sqr(ctx0, x)), + lambda) + ); + + const res = c.ggml_opt(null, opt_params, f); + + try std.testing.expect(res == c.GGML_OPT_OK); + + const x_data_pointer: [*]f32 = @ptrCast(@alignCast(x.*.data)); + // print results + for (0..16) |i| { + std.debug.print("x[{d:3}] = {d:.6}\n", .{i, x_data_pointer[i]}); + } + std.debug.print("...\n", .{}); + for (NF - 16..NF) |i| { + std.debug.print("x[{d:3}] = {d:.6}\n", .{i, x_data_pointer[i]}); + } + std.debug.print("\n", .{}); + + for (0..NF) |i| { + if (i < NF/2) { + try std.testing.expect(is_close(x_data_pointer[i], 1.0, 1e-2)); + } else { + try std.testing.expect(is_close(x_data_pointer[i], -1.0, 1e-2)); + } + } + } + + _ = try std.io.getStdIn().reader().readByte(); +}