Skip to content

Commit ce9d074

Browse files
ggerganovbaramofme
ggerganov
authored and
baramofme
committed
metal : optimize concat kernel and fix set kernel threads (ggml-org#23411)
* metal : fix GGML_OP_SET kernel threads * tests : extend test_cpy to support different src/dst shapes Extend test_cpy to support different source and destination tensor shapes for CPY operations (reshaping), where the total number of elements must match. - Renamed ne -> ne_src, added ne_dst parameter (default: use src shape) - Added 50 new reshaping test cases covering 1D<->2D<->3D<->4D conversions - Tests exercise 1024 boundary, small shapes, and large dimensionality changes - Fixed dangling reference bug (storing & to temporary std::array) - Updated all existing test calls with permute/transpose args for compatibility Assisted-by: llama.cpp:local pi * metal : optimize concat kernel with row batching for small widths When ne0 < 256, batch multiple rows into a single threadgroup to improve occupancy. This avoids underutilizing the GPU when processing narrow tensors. - Dispatch nth = min(256, ne0) threads per group - Calculate nrptg (rows per threadgroup) to fill up to 256 threads - Update kernel index calculation to handle the row batching - Add boundary check for i1 >= ne1 Assisted-by: llama.cpp:local pi * tests : clean-up * tests : refactor CPY shape tests to use dimension permutations Replace 75 hardcoded test cases with a loop over permutations of {3, 5, 7, 32} (total elements: 3360). Each src permutation is tested against canonical sorted and reverse dst, skipping identical shapes. Covers F32, F16, and Q4_0 (when both src and dst ne0 == 32). Assisted-by: llama.cpp:local pi
1 parent 1f67129 commit ce9d074

3 files changed

Lines changed: 113 additions & 42 deletions

File tree

‎ggml/src/ggml-metal/ggml-metal-ops.cpp‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -597,9 +597,20 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) {
597597
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
598598
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
599599

600-
const int nth = std::min(1024, ne0);
600+
int nth = std::min(256, ne0);
601601

602-
ggml_metal_encoder_dispatch_threadgroups(enc, ne1, ne2, ne3, nth, 1, 1);
602+
// when rows are small, we can batch them together in a single threadgroup
603+
int nrptg = 1;
604+
if (nth < 256) {
605+
nrptg = std::min((256 + nth - 1) / nth, ne1);
606+
if (nrptg * nth > 256) {
607+
nrptg = 256 / nth;
608+
}
609+
}
610+
611+
const int nw0 = (ne1 + nrptg - 1) / nrptg;
612+
613+
ggml_metal_encoder_dispatch_threadgroups(enc, nw0, ne2, ne3, nth, nrptg, 1);
603614

604615
return 1;
605616
}
@@ -1854,7 +1865,7 @@ int ggml_metal_op_set(ggml_metal_op_t ctx, int idx) {
18541865
nk0 = ne10/ggml_blck_size(op->type);
18551866
}
18561867

1857-
int nth = std::min<int>(nk0, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
1868+
int nth = std::min<int>(nk0*ne11, 256);
18581869

18591870
// when rows are small, we can batch them together in a single threadgroup
18601871
int nrptg = 1;
@@ -1865,7 +1876,7 @@ int ggml_metal_op_set(ggml_metal_op_t ctx, int idx) {
18651876
nrptg = (nth + nk0 - 1)/nk0;
18661877
nth = nk0;
18671878

1868-
if (nrptg*nth > ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
1879+
if (nrptg*nth > 256) {
18691880
nrptg--;
18701881
}
18711882
}

‎ggml/src/ggml-metal/ggml-metal.metal‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9340,7 +9340,11 @@ kernel void kernel_concat(
93409340

93419341
const int i3 = tgpig.z;
93429342
const int i2 = tgpig.y;
9343-
const int i1 = tgpig.x;
9343+
const int i1 = ntg.y == 1 ? tgpig.x : tgpig.x*ntg.y + tpitg.y;
9344+
9345+
if (i1 >= args.ne1) {
9346+
return;
9347+
}
93449348

93459349
int o[4] = {0, 0, 0, 0};
93469350
o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03));

‎tests/test-backend-ops.cpp‎

Lines changed: 93 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2871,15 +2871,24 @@ struct test_set : public test_case {
28712871
struct test_cpy : public test_case {
28722872
const ggml_type type_src;
28732873
const ggml_type type_dst;
2874-
const std::array<int64_t, 4> ne;
2874+
const std::array<int64_t, 4> ne_src;
2875+
const std::array<int64_t, 4> ne_dst;
28752876
const std::array<int64_t, 4> permute_src;
28762877
const std::array<int64_t, 4> permute_dst;
28772878
bool _src_use_permute;
28782879
bool _dst_use_permute;
28792880
bool _src_transpose;
2881+
bool _use_dst_shape;
28802882

28812883
std::string vars() override {
2882-
return VARS_TO_STR6(type_src, type_dst, ne, permute_src, permute_dst, _src_transpose);
2884+
if (_use_dst_shape) {
2885+
return VARS_TO_STR7(type_src, type_dst, ne_src, ne_dst, permute_src, permute_dst, _src_transpose);
2886+
}
2887+
return VARS_TO_STR6(type_src, type_dst, ne_src, permute_src, permute_dst, _src_transpose);
2888+
}
2889+
2890+
int64_t total_elements() const {
2891+
return ne_src[0] * ne_src[1] * ne_src[2] * ne_src[3];
28832892
}
28842893

28852894
double max_nmse_err() override {
@@ -2904,7 +2913,7 @@ struct test_cpy : public test_case {
29042913
err_estimate /= 8.0f;
29052914
}
29062915
err_estimate *= err_estimate;
2907-
err_estimate /= (150.0f*150.0f*0.25f)*float(ne[0] * ne[1] * ne[2] * ne[3]);
2916+
err_estimate /= (150.0f*150.0f*0.25f)*float(total_elements());
29082917
return err_estimate;
29092918
}
29102919
return 1e-6;
@@ -2915,17 +2924,19 @@ struct test_cpy : public test_case {
29152924
}
29162925

29172926
test_cpy(ggml_type type_src = GGML_TYPE_F32, ggml_type type_dst = GGML_TYPE_F32,
2918-
std::array<int64_t, 4> ne = {10, 10, 10, 1},
2927+
std::array<int64_t, 4> ne_src = {10, 10, 10, 1},
2928+
std::array<int64_t, 4> ne_dst = {-1, -1, -1, -1},
29192929
std::array<int64_t, 4> permute_src = {0, 0, 0, 0},
29202930
std::array<int64_t, 4> permute_dst = {0, 0, 0, 0},
29212931
bool transpose_src = false)
2922-
: type_src(type_src), type_dst(type_dst), ne(ne), permute_src(permute_src), permute_dst(permute_dst),
2932+
: type_src(type_src), type_dst(type_dst), ne_src(ne_src), ne_dst(ne_dst), permute_src(permute_src), permute_dst(permute_dst),
29232933
_src_use_permute(permute_src[0] + permute_src[1] + permute_src[2] + permute_src[3] > 0),
29242934
_dst_use_permute(permute_dst[0] + permute_dst[1] + permute_dst[2] + permute_dst[3] > 0),
2925-
_src_transpose(transpose_src){}
2935+
_src_transpose(transpose_src),
2936+
_use_dst_shape(ne_dst[0] >= 0 && ne_dst[1] >= 0 && ne_dst[2] >= 0 && ne_dst[3] >= 0){}
29262937

29272938
ggml_tensor * build_graph(ggml_context * ctx) override {
2928-
ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne.data());
2939+
ggml_tensor * src = ggml_new_tensor(ctx, type_src, 4, ne_src.data());
29292940
ggml_set_param(src);
29302941
ggml_set_name(src, "src");
29312942

@@ -2939,7 +2950,8 @@ struct test_cpy : public test_case {
29392950
ggml_set_name(src, "src_transposed");
29402951
}
29412952

2942-
ggml_tensor * dst = ggml_new_tensor(ctx, type_dst, 4, src->ne);
2953+
std::array<int64_t, 4> dst_ne = _use_dst_shape ? ne_dst : std::array<int64_t, 4>{src->ne[0], src->ne[1], src->ne[2], src->ne[3]};
2954+
ggml_tensor * dst = ggml_new_tensor(ctx, type_dst, 4, dst_ne.data());
29432955
ggml_set_name(dst, "dst");
29442956

29452957
if (_dst_use_permute) {
@@ -8228,42 +8240,72 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
82288240

82298241
for (int k = 1; k < 4; ++k) {
82308242
test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}));
8231-
test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {0, 2, 1, 3}));
8232-
test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {0, 3, 1, 2}, {0, 2, 1, 3}));
8243+
test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3}));
8244+
test_cases.emplace_back(new test_cpy(type, type, {k*nk, 2, 3, 4}, {-1,-1,-1,-1}, {0, 3, 1, 2}, {0, 2, 1, 3}));
82338245
}
82348246
}
82358247

82368248
for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
82378249
for (ggml_type type_dst : all_types) {
82388250
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4}));
8239-
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {0, 2, 1, 3})); // cpy by rows
8251+
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows
82408252
}
82418253
}
82428254
for (ggml_type type_src : all_types) {
82438255
for (ggml_type type_dst : {GGML_TYPE_F32}) {
82448256
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 4, 4, 4}));
8245-
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {0, 2, 1, 3})); // cpy by rows
8257+
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // cpy by rows
82468258
}
82478259
}
82488260
for (ggml_type type_src : {GGML_TYPE_F16, GGML_TYPE_F32}) {
82498261
for (ggml_type type_dst : {GGML_TYPE_F16, GGML_TYPE_F32}) {
8250-
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {1, 0, 2, 3})); // cpy not-contiguous
8262+
test_cases.emplace_back(new test_cpy(type_src, type_dst, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3})); // cpy not-contiguous
82518263
}
82528264
}
82538265
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}));
8254-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {1, 0, 2, 3}));
8266+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_I32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3}));
82558267
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}));
8256-
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}, {1, 0, 2, 3}));
8257-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8258-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8259-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 3}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8260-
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8261-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8262-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8263-
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8264-
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 4, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8265-
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
8266-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
8268+
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_F32, {256, 2, 3, 4}, {-1,-1,-1,-1}, {1, 0, 2, 3}));
8269+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8270+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8271+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 3, 3}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8272+
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 3, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8273+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8274+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8275+
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8276+
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 4, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
8277+
test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
8278+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0}));
8279+
8280+
// CPY - different src/dst shapes (reshaping via CPY)
8281+
// Use permutations of {3, 5, 7, 32}. Total elements: 3*5*7*32 = 3360.
8282+
// Each src permutation is tested against canonical sorted and reverse dst (skip self).
8283+
{
8284+
std::array<int64_t, 4> dims = {3, 5, 7, 32};
8285+
std::sort(dims.begin(), dims.end());
8286+
std::array<int64_t, 4> canonical = dims;
8287+
std::array<int64_t, 4> reversed = {32, 7, 5, 3};
8288+
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
8289+
std::array<int64_t, 4> cur = dims;
8290+
do {
8291+
if (cur != canonical) {
8292+
test_cases.emplace_back(new test_cpy(type, type, cur, canonical));
8293+
}
8294+
if (cur != reversed) {
8295+
test_cases.emplace_back(new test_cpy(type, type, cur, reversed));
8296+
}
8297+
if (cur[0] == 32 && type == GGML_TYPE_F32) {
8298+
if (canonical[0] == 32) {
8299+
test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, canonical));
8300+
}
8301+
if (reversed[0] == 32) {
8302+
test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, cur, reversed));
8303+
}
8304+
}
8305+
std::next_permutation(cur.begin(), cur.end());
8306+
} while (cur != canonical);
8307+
}
8308+
}
82678309

82688310
for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_I32, GGML_TYPE_F16, GGML_TYPE_BF16 }) {
82698311
for (bool use_view_slice : { true, false }) {
@@ -9043,9 +9085,24 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
90439085
test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {256, 16, 2, 3}, 1));
90449086
test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {128, 16, 2, 3}, 2));
90459087
test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 2, 3}, {64, 16, 2, 3}, 3));
9088+
90469089
test_cases.emplace_back(new test_pad());
90479090
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {33, 17, 2, 1}, 4, 3, true)); // circular
90489091
test_cases.emplace_back(new test_pad_ext());
9092+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1024, 1, 1, 1}, 1, 0, false));
9093+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1024, 2, 1, 1}, 1, 0, false));
9094+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1024, 16, 1, 1}, 0, 1, false));
9095+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1023, 1, 1, 1}, 1, 0, false));
9096+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1023, 8, 1, 1}, 1, 0, false));
9097+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1025, 1, 1, 1}, 1, 0, false));
9098+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {1025, 8, 1, 1}, 1, 0, false));
9099+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {2048, 1, 1, 1}, 1, 0, false));
9100+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {2048, 4, 1, 1}, 1, 0, false));
9101+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {2049, 1, 1, 1}, 1, 0, false));
9102+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 1, 1, 1}, 100, 0, false));
9103+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 1, 1, 1}, 0, 100, false));
9104+
test_cases.emplace_back(new test_pad(GGML_TYPE_F32, {100, 100, 1, 1}, 50, 50, false));
9105+
90499106
test_cases.emplace_back(new test_pad_reflect_1d());
90509107
test_cases.emplace_back(new test_pad_reflect_1d(GGML_TYPE_F32, {3000, 384, 4, 1}));
90519108
test_cases.emplace_back(new test_roll());
@@ -9365,22 +9422,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
93659422
test_cases.emplace_back(new test_bin_bcast(ggml_add, GGML_TYPE_F32, {4096, 1, 1, 1}, {1, 512, 1, 1}));
93669423

93679424
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F16, {512, 3072, 1, 1}));
9368-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {8192, 512, 2, 1}, {0, 2, 1, 3}));
9369-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {3072, 512, 2, 1}, {0, 2, 1, 3}));
9425+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {8192, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3}));
9426+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {3072, 512, 2, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3}));
93709427
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_Q4_0, {8192, 512, 2, 1}));
93719428
test_cases.emplace_back(new test_cpy(GGML_TYPE_Q4_0, GGML_TYPE_F32, {8192, 512, 2, 1}));
93729429

9373-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9374-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9375-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9376-
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9377-
9378-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9379-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9380-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9381-
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9382-
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9430+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9431+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9432+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
9433+
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3}, {0, 0, 0, 0}));
93839434

9435+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9436+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9437+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768*1024, 256, 1, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9438+
test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
9439+
test_cases.emplace_back(new test_cpy(GGML_TYPE_BF16, GGML_TYPE_BF16, {768, 1024, 256, 1}, {-1,-1,-1,-1}, {0, 0, 0, 0}, {0, 0, 0, 0}, true));
93849440

93859441
test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {4096, 4096, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));
93869442
test_cases.emplace_back(new test_soft_max(GGML_TYPE_F32, {12888, 256, 5, 1}, false, false, GGML_TYPE_F32, {1, 1}, 1.0f, 0.0f));

0 commit comments

Comments
 (0)