Skip to content

Commit a6ab3d4

Browse files
l3utterflyl3utterfly
authored
Merge branch 'layla-build' into merge
2 parents 80d0d6b + 14508d7 commit a6ab3d4

29 files changed

+774748
-142
lines changed

‎.gitignore‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ autogen-*.md
7575
!.github/workflows/*.yml
7676

7777
# Models
78-
7978
models/*
8079
models-mnt
8180
!models/.editorconfig

‎CMakeLists.txt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,3 +243,21 @@ configure_file(cmake/llama.pc.in
243243

244244
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/llama.pc"
245245
DESTINATION lib/pkgconfig)
246+
247+
#
248+
# utils, programs, examples and tests
249+
#
250+
251+
if (LLAMA_BUILD_COMMON)
252+
add_subdirectory(common)
253+
endif()
254+
255+
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)
256+
include(CTest)
257+
add_subdirectory(tests)
258+
endif()
259+
260+
if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_EXAMPLES)
261+
add_subdirectory(examples)
262+
add_subdirectory(pocs)
263+
endif()

‎common/common.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1038,6 +1038,9 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
10381038
auto mparams = llama_model_default_params();
10391039

10401040
if (!params.devices.empty()) {
1041+
// add nullptr to the end just in case
1042+
params.devices.push_back(nullptr);
1043+
10411044
mparams.devices = params.devices.data();
10421045
}
10431046
if (params.n_gpu_layers != -1) {
@@ -2072,4 +2075,3 @@ common_control_vector_data common_control_vector_load(const std::vector<common_c
20722075

20732076
return result;
20742077
}
2075-

‎common/sampling.cpp‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,17 @@ struct ring_buffer {
6060
return value;
6161
}
6262

63+
T pop_back() {
64+
if (sz == 0) {
65+
throw std::runtime_error("ring buffer is empty");
66+
}
67+
// Move pos backwards, wrapping around if necessary
68+
pos = (pos == 0) ? capacity - 1 : pos - 1;
69+
T value = data[pos];
70+
sz--;
71+
return value;
72+
}
73+
6374
const T & rat(size_t i) const {
6475
if (i >= sz) {
6576
throw std::runtime_error("ring buffer: index out of bounds");
@@ -248,6 +259,12 @@ void common_sampler_reset(struct common_sampler * gsmpl) {
248259
llama_sampler_reset(gsmpl->chain);
249260
}
250261

262+
void common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar) {
263+
llama_sampler_reset(gsmpl->grmr);
264+
265+
gsmpl->grmr = llama_sampler_init_grammar(llama_model_get_vocab(model), grammar, "root");
266+
}
267+
251268
struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {
252269
return new common_sampler {
253270
/* .params = */ gsmpl->params,
@@ -401,6 +418,21 @@ std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx_
401418
return result;
402419
}
403420

421+
const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl) {
422+
return gsmpl->prev.to_vector();
423+
}
424+
425+
void common_sampler_rollback(common_sampler * gsmpl, int rollback_num) {
426+
if(rollback_num > gsmpl->prev.size()) {
427+
rollback_num = gsmpl->prev.size();
428+
}
429+
430+
// continuously pop the last token
431+
for(int i = 0; i < rollback_num; i++) {
432+
gsmpl->prev.pop_back();
433+
}
434+
}
435+
404436
char common_sampler_type_to_chr(enum common_sampler_type cnstr) {
405437
switch (cnstr) {
406438
case COMMON_SAMPLER_TYPE_DRY: return 'd';

‎common/sampling.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ void common_sampler_free(struct common_sampler * gsmpl);
4343
// if accept_grammar is true, the token is accepted both by the sampling chain and the grammar
4444
void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool accept_grammar);
4545
void common_sampler_reset (struct common_sampler * gsmpl);
46+
void common_sampler_reinit_grammar(struct common_sampler * gsmpl, const struct llama_model * model, const char * grammar);
4647
struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl);
4748

4849
// arguments can be nullptr to skip printing
@@ -96,6 +97,8 @@ std::string common_sampler_print(const struct common_sampler * gsmpl);
9697

9798
// get a string representation of the last accepted tokens
9899
std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx, int n);
100+
const std::vector<llama_token> common_sampler_prev(common_sampler * gsmpl);
101+
void common_sampler_rollback(common_sampler * gsmpl, int rollback_num);
99102

100103
char common_sampler_type_to_chr(enum common_sampler_type cnstr);
101104
std::string common_sampler_type_to_str(enum common_sampler_type cnstr);

‎ggml/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,13 @@ set(GGML_PUBLIC_HEADERS
250250
include/gguf.h)
251251

252252
set_target_properties(ggml PROPERTIES PUBLIC_HEADER "${GGML_PUBLIC_HEADERS}")
253+
254+
# link android log library
255+
if(ANDROID)
256+
find_library(log-lib log)
257+
target_link_libraries(ggml PRIVATE ${log-lib})
258+
endif()
259+
253260
#if (GGML_METAL)
254261
# set_target_properties(ggml PROPERTIES RESOURCE "${CMAKE_CURRENT_SOURCE_DIR}/src/ggml-metal.metal")
255262
#endif()

‎ggml/include/ggml-backend.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ extern "C" {
202202
//
203203
// Backend registry
204204
//
205+
GGML_API void ggml_backend_reg_layla(bool useVulkan, bool useOpenCL);
205206

206207
GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
207208

‎ggml/src/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,14 @@ add_library(ggml-base
219219
ggml-quants.h
220220
gguf.cpp)
221221

222+
# Search for the 'log' library on Android
223+
if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android")
224+
find_library(log-lib log)
225+
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${log-lib})
226+
227+
target_link_libraries(ggml-base PUBLIC ${GGML_EXTRA_LIBS})
228+
endif()
229+
222230
target_include_directories(ggml-base PRIVATE .)
223231

224232
add_library(ggml

‎ggml/src/ggml-backend-reg.cpp‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,9 @@ struct ggml_backend_reg_entry {
148148
dl_handle_ptr handle;
149149
};
150150

151+
static bool laylaUseVulkan = false;
152+
static bool laylaUseOpenCL = false;
153+
151154
struct ggml_backend_registry {
152155
std::vector<ggml_backend_reg_entry> backends;
153156
std::vector<ggml_backend_dev_t> devices;
@@ -163,10 +166,14 @@ struct ggml_backend_registry {
163166
register_backend(ggml_backend_sycl_reg());
164167
#endif
165168
#ifdef GGML_USE_VULKAN
166-
register_backend(ggml_backend_vk_reg());
169+
if(laylaUseVulkan) {
170+
register_backend(ggml_backend_vk_reg());
171+
}
167172
#endif
168173
#ifdef GGML_USE_OPENCL
169-
register_backend(ggml_backend_opencl_reg());
174+
if(laylaUseOpenCL) {
175+
register_backend(ggml_backend_opencl_reg());
176+
}
170177
#endif
171178
#ifdef GGML_USE_CANN
172179
register_backend(ggml_backend_cann_reg());
@@ -288,6 +295,11 @@ struct ggml_backend_registry {
288295
}
289296
};
290297

298+
void ggml_backend_reg_layla(bool useVulkan, bool useOpenCL) {
299+
laylaUseVulkan = useVulkan;
300+
laylaUseOpenCL = useOpenCL;
301+
}
302+
291303
static ggml_backend_registry & get_reg() {
292304
static ggml_backend_registry reg;
293305
return reg;

‎ggml/src/ggml-blas/CMakeLists.txt‎

Lines changed: 81 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -5,83 +5,96 @@ endif()
55
# set(BLA_SIZEOF_INTEGER 8)
66
#endif()
77

8-
set(BLA_VENDOR ${GGML_BLAS_VENDOR})
9-
find_package(BLAS)
10-
11-
if (BLAS_FOUND)
12-
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
8+
if(ANDROID AND ${GGML_BLAS_VENDOR} MATCHES "Generic")
9+
message(STATUS "Using local BLAS, Includes: ${BLAS_INCLUDE_DIRS}")
1310

1411
ggml_add_backend_library(ggml-blas
15-
ggml-blas.cpp
16-
)
12+
ggml-blas.cpp
13+
)
14+
15+
target_compile_options(ggml-blas PRIVATE ${BLAS_LINKER_FLAGS})
16+
17+
target_link_libraries (ggml-blas PRIVATE ${BLAS_LIBRARIES})
18+
target_include_directories(ggml-blas PRIVATE ${BLAS_INCLUDE_DIRS})
19+
else()
20+
set(BLA_VENDOR ${GGML_BLAS_VENDOR})
21+
find_package(BLAS)
22+
23+
if (BLAS_FOUND)
24+
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
1725

18-
if (${GGML_BLAS_VENDOR} MATCHES "Apple")
19-
add_compile_definitions(ACCELERATE_NEW_LAPACK)
20-
add_compile_definitions(ACCELERATE_LAPACK_ILP64)
21-
add_compile_definitions(GGML_BLAS_USE_ACCELERATE)
22-
elseif ("${BLAS_INCLUDE_DIRS}" STREQUAL "")
23-
# BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
24-
# see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
25-
find_package(PkgConfig REQUIRED)
26-
if (${GGML_BLAS_VENDOR} MATCHES "Generic")
27-
pkg_check_modules(DepBLAS blas)
28-
elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
29-
# As of openblas v0.3.22, the 64-bit is named openblas64.pc
30-
pkg_check_modules(DepBLAS openblas64)
31-
if (NOT DepBLAS_FOUND)
32-
pkg_check_modules(DepBLAS openblas)
26+
ggml_add_backend_library(ggml-blas
27+
ggml-blas.cpp
28+
)
29+
30+
if (${GGML_BLAS_VENDOR} MATCHES "Apple")
31+
add_compile_definitions(ACCELERATE_NEW_LAPACK)
32+
add_compile_definitions(ACCELERATE_LAPACK_ILP64)
33+
add_compile_definitions(GGML_BLAS_USE_ACCELERATE)
34+
elseif ("${BLAS_INCLUDE_DIRS}" STREQUAL "")
35+
# BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
36+
# see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
37+
find_package(PkgConfig REQUIRED)
38+
if (${GGML_BLAS_VENDOR} MATCHES "Generic")
39+
pkg_check_modules(DepBLAS blas)
40+
elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
41+
# As of openblas v0.3.22, the 64-bit is named openblas64.pc
42+
pkg_check_modules(DepBLAS openblas64)
43+
if (NOT DepBLAS_FOUND)
44+
pkg_check_modules(DepBLAS openblas)
45+
endif()
46+
elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
47+
add_compile_definitions(GGML_BLAS_USE_BLIS)
48+
pkg_check_modules(DepBLAS blis)
49+
elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
50+
pkg_check_modules(DepBLAS blas-atlas)
51+
elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
52+
pkg_check_modules(DepBLAS flexiblas_api)
53+
elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
54+
add_compile_definitions(GGML_BLAS_USE_MKL)
55+
# all Intel* libraries share the same include path
56+
pkg_check_modules(DepBLAS mkl-sdl)
57+
elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
58+
# this doesn't provide pkg-config
59+
# suggest to assign BLAS_INCLUDE_DIRS on your own
60+
if ("${NVHPC_VERSION}" STREQUAL "")
61+
message(WARNING "Better to set NVHPC_VERSION")
62+
else()
63+
set(DepBLAS_FOUND ON)
64+
set(DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include")
65+
endif()
3366
endif()
34-
elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
35-
add_compile_definitions(GGML_BLAS_USE_BLIS)
36-
pkg_check_modules(DepBLAS blis)
37-
elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
38-
pkg_check_modules(DepBLAS blas-atlas)
39-
elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
40-
pkg_check_modules(DepBLAS flexiblas_api)
41-
elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
42-
add_compile_definitions(GGML_BLAS_USE_MKL)
43-
# all Intel* libraries share the same include path
44-
pkg_check_modules(DepBLAS mkl-sdl)
45-
elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
46-
# this doesn't provide pkg-config
47-
# suggest to assign BLAS_INCLUDE_DIRS on your own
48-
if ("${NVHPC_VERSION}" STREQUAL "")
49-
message(WARNING "Better to set NVHPC_VERSION")
67+
if (DepBLAS_FOUND)
68+
set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})
5069
else()
51-
set(DepBLAS_FOUND ON)
52-
set(DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include")
70+
message(WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
71+
" detected by pkgconfig, trying to find cblas.h from possible paths...")
72+
find_path(BLAS_INCLUDE_DIRS
73+
NAMES cblas.h
74+
HINTS
75+
/usr/include
76+
/usr/local/include
77+
/usr/include/openblas
78+
/opt/homebrew/opt/openblas/include
79+
/usr/local/opt/openblas/include
80+
/usr/include/x86_64-linux-gnu/openblas/include
81+
)
5382
endif()
5483
endif()
55-
if (DepBLAS_FOUND)
56-
set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})
57-
else()
58-
message(WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
59-
" detected by pkgconfig, trying to find cblas.h from possible paths...")
60-
find_path(BLAS_INCLUDE_DIRS
61-
NAMES cblas.h
62-
HINTS
63-
/usr/include
64-
/usr/local/include
65-
/usr/include/openblas
66-
/opt/homebrew/opt/openblas/include
67-
/usr/local/opt/openblas/include
68-
/usr/include/x86_64-linux-gnu/openblas/include
69-
)
70-
endif()
71-
endif()
7284

73-
message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}")
85+
message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}")
7486

75-
target_compile_options(ggml-blas PRIVATE ${BLAS_LINKER_FLAGS})
87+
target_compile_options(ggml-blas PRIVATE ${BLAS_LINKER_FLAGS})
7688

77-
if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
78-
add_compile_definitions(GGML_BLAS_USE_MKL)
79-
endif()
89+
if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
90+
add_compile_definitions(GGML_BLAS_USE_MKL)
91+
endif()
8092

81-
target_link_libraries (ggml-blas PRIVATE ${BLAS_LIBRARIES})
82-
target_include_directories(ggml-blas PRIVATE ${BLAS_INCLUDE_DIRS})
83-
else()
84-
message(ERROR "BLAS not found, please refer to "
85-
"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
86-
" to set correct GGML_BLAS_VENDOR")
93+
target_link_libraries (ggml-blas PRIVATE ${BLAS_LIBRARIES})
94+
target_include_directories(ggml-blas PRIVATE ${BLAS_INCLUDE_DIRS})
95+
else()
96+
message(ERROR "BLAS not found, please refer to "
97+
"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
98+
" to set correct GGML_BLAS_VENDOR")
99+
endif()
87100
endif()

0 commit comments

Comments
 (0)