Skip to content

Commit 2821acc

Browse files
author
zhouwg
committed
tests: a simple backend ut
1 parent f777a73 commit 2821acc

File tree

2 files changed

+341
-0
lines changed

2 files changed

+341
-0
lines changed

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ llama_target_and_test(test-chat-template.cpp)
137137
# llama_target_and_test(test-opt.cpp) # SLOW
138138
llama_target_and_test(test-gguf.cpp)
139139
llama_target_and_test(test-backend-ops.cpp)
140+
llama_target_and_test(simple-backend-ut.cpp)
140141

141142
llama_target_and_test(test-model-load-cancel.cpp LABEL "model")
142143
llama_target_and_test(test-autorelease.cpp LABEL "model")

‎tests/simple-backend-ut.cpp‎

Lines changed: 340 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
/*
2+
* Copyright (c) 2023-2024 The ggml authors
3+
*
4+
* Permission is hereby granted, free of charge, to any person obtaining a copy
5+
* of this software and associated documentation files (the "Software"), to
6+
* deal in the Software without restriction, including without limitation the
7+
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8+
* sell copies of the Software, and to permit persons to whom the Software is
9+
* furnished to do so, subject to the following conditions:
10+
*
11+
* The above copyright notice and this permission notice shall be included in
12+
* all copies or substantial portions of the Software.
13+
*
14+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19+
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20+
* IN THE SOFTWARE.
21+
*/
22+
#include <stdio.h>
23+
#include <stdlib.h>
24+
#include <stdint.h>
25+
#include <string.h>
26+
#include <stddef.h>
27+
#include <unistd.h>
28+
#include <inttypes.h>
29+
#include <math.h>
30+
#include <time.h>
31+
#include <unistd.h>
32+
#include <dlfcn.h>
33+
#include <fcntl.h>
34+
#include <sys/stat.h>
35+
#include <limits.h>
36+
#include <signal.h>
37+
#include <fcntl.h>
38+
#include <sys/types.h>
39+
40+
#include <string>
41+
#include <vector>
42+
#include <thread>
43+
#include <mutex>
44+
#include <map>
45+
#include <set>
46+
#include <tuple>
47+
#include <queue>
48+
#include <fstream>
49+
#include <iostream>
50+
#include <iomanip>
51+
#include <sstream>
52+
#include <chrono>
53+
#include <memory>
54+
#include <regex>
55+
#include <random>
56+
#include <functional>
57+
#include <unordered_map>
58+
#include <condition_variable>
59+
#include <cassert>
60+
#include <unordered_set>
61+
#include <utility>
62+
63+
#include "ggml.h"
64+
#include "ggml-cpu.h"
65+
#include "ggml-alloc.h"
66+
#include "ggml-backend.h"
67+
68+
#define LOG_BUF_LEN 4096
69+
#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor)
70+
71+
72+
static bool ggml_graph_compute_helper(
73+
struct ggml_backend * backend,
74+
struct ggml_cgraph * graph,
75+
std::vector<uint8_t> & buf,
76+
int n_threads,
77+
ggml_abort_callback abort_callback,
78+
void * abort_callback_data) {
79+
struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, NULL);
80+
81+
plan.abort_callback = abort_callback;
82+
plan.abort_callback_data = abort_callback_data;
83+
84+
if (plan.work_size > 0) {
85+
buf.resize(plan.work_size);
86+
plan.work_data = buf.data();
87+
}
88+
89+
if (nullptr != backend)
90+
return ggml_backend_graph_compute(backend, graph) == GGML_STATUS_SUCCESS;
91+
else
92+
return ggml_graph_compute(graph, &plan);
93+
}
94+
95+
96+
static void tensor_dump_elements(const ggml_tensor * tensor) {
97+
float value = 0;
98+
std::ostringstream tmposs;
99+
if (tensor->type == GGML_TYPE_F32) {
100+
for (int h = 0; h < tensor->ne[3]; h++) {
101+
for (int i = 0; i < tensor->ne[2]; i++) {
102+
for (int j = 0; j < tensor->ne[1]; j++) {
103+
for (int k = 0; k < tensor->ne[0]; k++) {
104+
value = ((float *) tensor->data)[h * tensor->ne[2] + i * tensor->ne[1] +
105+
j * tensor->ne[0] + k];
106+
tmposs << std::setw(8) << std::fixed << std::setprecision(2) << value
107+
<< " ";
108+
}
109+
if (strlen(tmposs.str().c_str()) <= (LOG_BUF_LEN - 96)) {
110+
printf("%s\n", tmposs.str().c_str());
111+
}
112+
tmposs.clear();
113+
tmposs.str("");
114+
}
115+
}
116+
}
117+
}
118+
119+
printf("\n");
120+
}
121+
122+
123+
static void tensor_dump(const ggml_tensor * tensor, const char * name) {
124+
printf("dump ggml tensor %s(%s)\n", name, tensor->name);
125+
printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi)\n",
126+
name,
127+
tensor->type, ggml_type_name(tensor->type),
128+
tensor->ne[0], tensor->ne[1], tensor->ne[2],
129+
tensor->nb[0], tensor->nb[1], tensor->nb[2]);
130+
tensor_dump_elements(tensor);
131+
132+
printf("\n");
133+
}
134+
135+
136+
static uint32_t get_tensor_rank(const ggml_tensor * tensor) {
137+
uint32_t rank = 0;
138+
for (int i = 0; i < GGML_MAX_DIMS; i++) {
139+
if ((0 != tensor->ne[i]) && (1 != tensor->ne[i])) {
140+
rank++;
141+
}
142+
}
143+
return rank;
144+
}
145+
146+
147+
static uint32_t get_tensor_data_size(const ggml_tensor * tensor) {
148+
size_t data_size = ggml_row_size(tensor->type, tensor->ne[0]);
149+
size_t n_dims = get_tensor_rank(tensor);
150+
for (size_t i = 1; i < n_dims; i++) {
151+
data_size *= tensor->ne[i];
152+
}
153+
printf("get_tensor_data_size %ld", data_size);
154+
printf("ggml_nbytes(tensor) %ld", ggml_nbytes(tensor));
155+
156+
return ggml_nbytes(tensor);
157+
}
158+
159+
160+
static void show_usage() {
161+
printf(" " \
162+
"\nUsage: simple-backend-ut [options]\n" \
163+
"\n" \
164+
"Options:\n" \
165+
" -t GGML_OP_ADD / GGML_OP_MUL / GGML_OP_MULMAT\n" \
166+
" ?/h print usage information\n\n"
167+
);
168+
}
169+
170+
171+
int main(int argc, char * argv[]) {
172+
int64_t n_begin_time = 0LL;
173+
int64_t n_end_time = 0LL;
174+
int64_t n_duration = 0LL;
175+
size_t ctx_size = 0;
176+
int sizey = 4;
177+
int sizex = 4;
178+
int num_threads = 4;
179+
int n_ggml_op_type = GGML_OP_ADD;
180+
181+
struct ggml_context * ctx = nullptr;
182+
struct ggml_cgraph * gf = nullptr;
183+
struct ggml_tensor * src0 = nullptr;
184+
struct ggml_tensor * src1 = nullptr;
185+
struct ggml_tensor * dst = nullptr;
186+
ggml_backend_t backend = nullptr;
187+
ggml_backend_buffer_t buffer= nullptr;
188+
ggml_type qtype = GGML_TYPE_F32;
189+
std::vector<uint8_t> work_buffer;
190+
191+
for (int i = 1; i < argc; i++) {
192+
if (0 == strcmp(argv[i], "-t")) {
193+
if (i + 1 < argc) {
194+
if (0 == memcmp(argv[i + 1], "GGML_OP_ADD", 11)) {
195+
n_ggml_op_type = GGML_OP_ADD;
196+
} else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL_MAT", 15)) {
197+
n_ggml_op_type = GGML_OP_MUL_MAT;
198+
} else if (0 == memcmp(argv[i + 1], "GGML_OP_MUL", 11)) {
199+
n_ggml_op_type = GGML_OP_MUL;
200+
} else {
201+
show_usage();
202+
return 1;
203+
}
204+
i++;
205+
}
206+
} else {
207+
show_usage();
208+
return 1;
209+
}
210+
}
211+
212+
printf("Testing %zu devices\n\n", ggml_backend_dev_count());
213+
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
214+
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
215+
216+
printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(),
217+
ggml_backend_dev_name(dev));
218+
219+
if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
220+
printf(" Skipping CPU backend\n");
221+
continue;
222+
}
223+
224+
backend = ggml_backend_dev_init(dev, reinterpret_cast<const char *>(i));
225+
GGML_ASSERT(backend != NULL);
226+
if (backend != nullptr) {
227+
printf("%s: initialize %s backend\n", __func__, ggml_backend_dev_name(dev));
228+
}
229+
230+
printf(" Device description: %s\n", ggml_backend_dev_description(dev));
231+
size_t free, total;
232+
ggml_backend_dev_memory(dev, &free, &total);
233+
printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);
234+
printf("\n");
235+
}
236+
237+
ggml_backend_t backend_cpu = nullptr;
238+
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
239+
if (nullptr == backend_cpu) {
240+
printf("failed to initialize cpu backend\n");
241+
exit(1);
242+
} else {
243+
printf("succeed to initialize cpu backend\n");
244+
}
245+
246+
printf("ggml op:%d(%s)", n_ggml_op_type, ggml_op_name((enum ggml_op) n_ggml_op_type));
247+
248+
n_begin_time = ggml_time_us();
249+
srand(time(NULL));
250+
251+
ctx_size += 1024 * 1024 * 32;
252+
printf("allocating Memory of size %zi bytes, %zi MB\n", ctx_size,
253+
(ctx_size / 1024 / 1024));
254+
255+
struct ggml_init_params params = {
256+
/*.mem_size =*/ ctx_size,
257+
/*.mem_buffer =*/ NULL,
258+
/* no_alloc =*/ 0
259+
};
260+
261+
ctx = ggml_init(params);
262+
if (!ctx) {
263+
printf("ggml_init() failed\n");
264+
return 2;
265+
}
266+
267+
printf("ggml_blck_size(%s) %ld\n", ggml_type_name(qtype), ggml_blck_size(qtype));
268+
printf("ggml_type_size(%s) %ld\n", ggml_type_name(qtype), ggml_type_size(qtype));
269+
if (qtype != GGML_TYPE_F32) {
270+
sizex = ggml_blck_size(qtype);
271+
}
272+
273+
printf("creating new tensors\n");
274+
src0 = ggml_new_tensor_2d(ctx, qtype, sizey, sizex);
275+
src1 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizey, sizex);
276+
277+
ggml_set_input(src0);
278+
ggml_set_input(src1);
279+
switch (n_ggml_op_type) {
280+
case GGML_OP_ADD:
281+
dst = ggml_add(ctx, src0, src1);
282+
break;
283+
case GGML_OP_MUL:
284+
dst = ggml_mul(ctx, src0, src1);
285+
break;
286+
case GGML_OP_MUL_MAT:
287+
dst = ggml_mul_mat(ctx, src0, src1);
288+
break;
289+
default:
290+
printf("ggml op %d(%s) not supported", n_ggml_op_type,
291+
ggml_op_name((enum ggml_op) n_ggml_op_type));
292+
ggml_free(ctx);
293+
ggml_backend_free(backend);
294+
ggml_backend_free(backend_cpu);
295+
return 3;
296+
}
297+
298+
ggml_set_output(dst);
299+
300+
printf("creating compute graph\n");
301+
gf = ggml_new_graph(ctx);
302+
ggml_build_forward_expand(gf, dst);
303+
304+
ggml_set_f32(src0, 1.0f);
305+
ggml_set_f32(src1, 2.0f);
306+
ggml_set_f32(dst, 0.0f);
307+
308+
ggml_graph_compute_helper(backend, gf, work_buffer, num_threads, nullptr, nullptr);
309+
if (get_tensor_data_size(dst) < (100 * 100)) {
310+
printf("dump result tensors:\n");
311+
TENSOR_DUMP(src0);
312+
TENSOR_DUMP(src1);
313+
TENSOR_DUMP(dst);
314+
} else {
315+
printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n",
316+
src0->name,
317+
src0->type, ggml_type_name(src0->type), src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
318+
src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);
319+
printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n",
320+
src1->name,
321+
src1->type, ggml_type_name(src1->type), src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3],
322+
src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);
323+
printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi, %5zi)\n",
324+
dst->name,
325+
dst->type, ggml_type_name(dst->type), dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], dst->nb[0],
326+
dst->nb[1], dst->nb[2], dst->nb[3]);
327+
}
328+
TENSOR_DUMP(dst);
329+
330+
ggml_free(ctx);
331+
ggml_backend_buffer_free(buffer);
332+
ggml_backend_free(backend);
333+
ggml_backend_free(backend_cpu);
334+
335+
n_end_time = ggml_time_us();
336+
n_duration = (n_end_time - n_begin_time) / 1000;
337+
printf("duration of ut GGML_OP_%s: %ld milliseconds\n", ggml_op_name((enum ggml_op)n_ggml_op_type), n_duration);
338+
339+
return 0;
340+
}

0 commit comments

Comments
 (0)