Skip to content

Commit 169738d

Browse files
slarenggerganov
slaren
authored and
ggerganov
committed
move BLAS to a separate backend (cont) (llama/6210)
ggml-ci
1 parent 47be086 commit 169738d

File tree

8 files changed

+118
-29
lines changed

8 files changed

+118
-29
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161

6262
- name: Configure CMake
6363
working-directory: ./build
64-
run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_TEST_COVERAGE=ON ..
64+
run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_TEST_COVERAGE=ON -DGGML_METAL=OFF ..
6565

6666
- name: Build
6767
working-directory: ./build
@@ -112,7 +112,7 @@ jobs:
112112

113113
- name: Configure CMake
114114
working-directory: ./build
115-
run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_TEST_COVERAGE=ON ..
115+
run: cmake -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DGGML_TEST_COVERAGE=ON -DGGML_METAL=OFF ..
116116

117117
- name: Build
118118
working-directory: ./build

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
build/
2+
build-blas/
23
build-debug/
34
build-release/
45
build-sanitize-addr/

‎CMakeLists.txt‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,16 @@ endif()
2525

2626
# options
2727

28+
if (APPLE)
29+
set(GGML_METAL_DEFAULT ON)
30+
set(GGML_BLAS_DEFAULT ON)
31+
set(GGML_BLAS_VENDOR_DEFAULT "Apple")
32+
else()
33+
set(GGML_METAL_DEFAULT OFF)
34+
set(GGML_BLAS_DEFAULT OFF)
35+
set(GGML_BLAS_VENDOR_DEFAULT "Generic")
36+
endif()
37+
2838
option(BUILD_SHARED_LIBS "ggml: build shared libs" ${BUILD_SHARED_LIBS_DEFAULT})
2939

3040
option(GGML_ALL_WARNINGS "ggml: enable all compiler warnings" ON)
@@ -41,11 +51,13 @@ option(GGML_TEST_COVERAGE "ggml: enable test coverage" OFF)
4151

4252
option(GGML_PERF "ggml: enable perf timings" OFF)
4353
option(GGML_NO_ACCELERATE "ggml: disable Accelerate framework" OFF)
44-
option(GGML_OPENBLAS "ggml: use OpenBLAS" OFF)
54+
option(GGML_BLAS "ggml: use BLAS" ${GGML_BLAS_DEFAULT})
55+
set(GGML_BLAS_VENDOR ${GGML_BLAS_VENDOR_DEFAULT} CACHE STRING
56+
"ggml: BLAS library vendor")
4557
option(GGML_HIPBLAS "ggml: use hipBLAS" OFF)
4658
option(GGML_CUDA "ggml: use CUDA" OFF)
4759
option(GGML_CUBLAS "ggml: use CUDA (deprecated)" OFF)
48-
option(GGML_METAL "ggml: use Metal" OFF)
60+
option(GGML_METAL "ggml: use Metal" ${GGML_METAL_DEFAULT})
4961
option(GGML_METAL_NDEBUG "ggml: disable Metal debugging" OFF)
5062
option(GGML_METAL_SHADER_DEBUG "ggml: compile Metal with -fno-fast-math" OFF)
5163
option(GGML_METAL_EMBED_LIBRARY "ggml: embed Metal library" OFF)

‎examples/common.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ struct gpt_params {
2121
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
2222
int32_t n_predict = 200; // new tokens to predict
2323
int32_t n_parallel = 1; // number of parallel streams
24-
int32_t n_batch = 8; // batch size for prompt processing
24+
int32_t n_batch = 32; // batch size for prompt processing
2525
int32_t n_ctx = 2048; // context size (this is the KV cache max size)
2626
int32_t n_gpu_layers = 0; // number of layers to offlload to the GPU
2727

‎examples/gpt-2/main-sched.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@
1010
#include "ggml-metal.h"
1111
#endif
1212

13+
#ifdef GGML_USE_BLAS
14+
#include "ggml-blas.h"
15+
#endif
16+
1317
#include "common.h"
1418
#include "common-ggml.h"
1519

@@ -131,6 +135,16 @@ void init_backends(gpt2_model & model, const gpt_params & params) {
131135
model.backends.push_back(gpu_backend);
132136
}
133137

138+
#ifdef GGML_USE_BLAS
139+
ggml_backend_t blas_backend = ggml_backend_blas_init();
140+
if (!blas_backend) {
141+
fprintf(stderr, "%s: failed to initialize BLAS backend\n", __func__);
142+
} else {
143+
ggml_backend_blas_set_n_threads(blas_backend, params.n_threads);
144+
model.backends.push_back(blas_backend);
145+
}
146+
#endif
147+
134148
// always add the CPU backend as a fallback
135149
ggml_backend_t cpu_backend = ggml_backend_cpu_init();
136150
ggml_backend_cpu_set_n_threads(cpu_backend, params.n_threads);

‎src/CMakeLists.txt‎

Lines changed: 86 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -152,28 +152,89 @@ if (APPLE AND NOT GGML_NO_ACCELERATE)
152152
endif()
153153
endif()
154154

155-
if (GGML_OPENBLAS)
156-
set(OPENBLAS_INCLUDE_SEARCH_PATHS
157-
/usr/include
158-
/usr/include/openblas
159-
/usr/include/openblas-base
160-
/usr/local/include
161-
/usr/local/include/openblas
162-
/usr/local/include/openblas-base
163-
/opt/OpenBLAS/include
164-
$ENV{OpenBLAS_HOME}
165-
$ENV{OpenBLAS_HOME}/include
166-
)
167-
find_path(OPENBLAS_INC NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS})
168-
find_library(OPENBLAS_LIB NAMES openblas libopenblas)
169-
if (OPENBLAS_LIB)
170-
message(STATUS "OpenBLAS found")
171-
172-
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${OPENBLAS_LIB})
173-
set(GGML_EXTRA_INCS ${GGML_EXTRA_INCS} ${OPENBLAS_INC})
174-
set(GGML_EXTRA_FLAGS ${GGML_EXTRA_FLAGS} -DGGML_USE_OPENBLAS)
155+
if (GGML_BLAS)
156+
if (GGML_STATIC)
157+
set(BLA_STATIC ON)
158+
endif()
159+
#if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.22)
160+
# set(BLA_SIZEOF_INTEGER 8)
161+
#endif()
162+
163+
set(BLA_VENDOR ${GGML_BLAS_VENDOR})
164+
find_package(BLAS)
165+
166+
if (BLAS_FOUND)
167+
message(STATUS "BLAS found, Libraries: ${BLAS_LIBRARIES}")
168+
169+
if (("${BLAS_INCLUDE_DIRS}" STREQUAL "") AND NOT (${GGML_BLAS_VENDOR} MATCHES "Apple"))
170+
# BLAS_INCLUDE_DIRS is missing in FindBLAS.cmake.
171+
# see https://gitlab.kitware.com/cmake/cmake/-/issues/20268
172+
find_package(PkgConfig REQUIRED)
173+
if (${GGML_BLAS_VENDOR} MATCHES "Generic")
174+
pkg_check_modules(DepBLAS REQUIRED blas)
175+
elseif (${GGML_BLAS_VENDOR} MATCHES "OpenBLAS")
176+
# As of openblas v0.3.22, the 64-bit is named openblas64.pc
177+
pkg_check_modules(DepBLAS openblas64)
178+
if (NOT DepBLAS_FOUND)
179+
pkg_check_modules(DepBLAS REQUIRED openblas)
180+
endif()
181+
elseif (${GGML_BLAS_VENDOR} MATCHES "FLAME")
182+
pkg_check_modules(DepBLAS REQUIRED blis)
183+
elseif (${GGML_BLAS_VENDOR} MATCHES "ATLAS")
184+
pkg_check_modules(DepBLAS REQUIRED blas-atlas)
185+
elseif (${GGML_BLAS_VENDOR} MATCHES "FlexiBLAS")
186+
pkg_check_modules(DepBLAS REQUIRED flexiblas_api)
187+
elseif (${GGML_BLAS_VENDOR} MATCHES "Intel")
188+
# all Intel* libraries share the same include path
189+
pkg_check_modules(DepBLAS REQUIRED mkl-sdl)
190+
elseif (${GGML_BLAS_VENDOR} MATCHES "NVHPC")
191+
# this doesn't provide pkg-config
192+
# suggest to assign BLAS_INCLUDE_DIRS on your own
193+
if ("${NVHPC_VERSION}" STREQUAL "")
194+
message(WARNING "Better to set NVHPC_VERSION")
195+
else()
196+
set(DepBLAS_FOUND ON)
197+
set(DepBLAS_INCLUDE_DIRS "/opt/nvidia/hpc_sdk/${CMAKE_SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR}/${NVHPC_VERSION}/math_libs/include")
198+
endif()
199+
endif()
200+
if (DepBLAS_FOUND)
201+
set(BLAS_INCLUDE_DIRS ${DepBLAS_INCLUDE_DIRS})
202+
else()
203+
message(WARNING "BLAS_INCLUDE_DIRS neither been provided nor been automatically"
204+
" detected by pkgconfig, trying to find cblas.h from possible paths...")
205+
find_path(BLAS_INCLUDE_DIRS
206+
NAMES cblas.h
207+
HINTS
208+
/usr/include
209+
/usr/local/include
210+
/usr/include/openblas
211+
/opt/homebrew/opt/openblas/include
212+
/usr/local/opt/openblas/include
213+
/usr/include/x86_64-linux-gnu/openblas/include
214+
)
215+
endif()
216+
endif()
217+
218+
message(STATUS "BLAS found, Includes: ${BLAS_INCLUDE_DIRS}")
219+
220+
add_compile_options(${BLAS_LINKER_FLAGS})
221+
222+
add_compile_definitions(GGML_USE_BLAS)
223+
224+
if (${BLAS_INCLUDE_DIRS} MATCHES "mkl" AND (${GGML_BLAS_VENDOR} MATCHES "Generic" OR ${GGML_BLAS_VENDOR} MATCHES "Intel"))
225+
add_compile_definitions(GGML_BLAS_USE_MKL)
226+
endif()
227+
228+
set(GGML_HEADERS_BLAS ggml-blas.h)
229+
set(GGML_SOURCES_BLAS ggml-blas.cpp)
230+
231+
set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${BLAS_LIBRARIES})
232+
set(GGML_EXTRA_INCLUDES ${GGML_EXTRA_INCLUDES} ${BLAS_INCLUDE_DIRS})
233+
set(GGML_EXTRA_FLAGS ${GGML_EXTRA_FLAGS} -DGGML_USE_BLAS)
175234
else()
176-
message(WARNING "OpenBLAS not found")
235+
message(WARNING "BLAS not found, please refer to "
236+
"https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors"
237+
" to set correct GGML_BLAS_VENDOR")
177238
endif()
178239
endif()
179240

@@ -513,9 +574,10 @@ add_library(${TARGET}
513574
../include/ggml/ggml.h
514575
../include/ggml/ggml-alloc.h
515576
../include/ggml/ggml-backend.h
516-
${GGML_SOURCES_CUDA}
517-
${GGML_SOURCES_METAL}
518-
${GGML_SOURCES_RPC}
577+
${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA}
578+
${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL}
579+
${GGML_SOURCES_RPC} ${GGML_HEADERS_RPC}
580+
${GGML_SOURCES_BLAS} ${GGML_HEADERS_BLAS}
519581
)
520582

521583
target_include_directories(${TARGET} PUBLIC

‎ggml-blas.cpp‎ renamed to ‎src/ggml-blas.cpp‎

File renamed without changes.

‎ggml-blas.h‎ renamed to ‎src/ggml-blas.h‎

File renamed without changes.

0 commit comments

Comments
 (0)