Skip to content

Commit bac7868

Browse files
slarenslaren
committed
add amx, cann, sycl
1 parent 646e91a commit bac7868

File tree

6 files changed

+261
-195
lines changed

6 files changed

+261
-195
lines changed

‎ggml/src/CMakeLists.txt‎

Lines changed: 21 additions & 194 deletions
Original file line numberDiff line numberDiff line change
@@ -29,27 +29,6 @@ endif()
2929
unset(GGML_EXTRA_LIBS_PRIVATE)
3030
unset(GGML_EXTRA_LIBS_PUBLIC)
3131

32-
if (GGML_AMX)
33-
if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.0)
34-
else()
35-
set(GGML_AMX OFF)
36-
message(WARNING "AMX requires gcc version > 11.0. Turning off GGML_AMX.")
37-
endif()
38-
39-
if (GGML_AMX)
40-
message(STATUS "Using AMX")
41-
42-
list(APPEND GGML_CDEF_PUBLIC GGML_USE_AMX)
43-
44-
file(GLOB GGML_HEADERS_AMX "ggml-amx/*.h")
45-
list(APPEND GGML_HEADERS_AMX "../include/ggml-amx.h")
46-
47-
file(GLOB GGML_SOURCES_AMX "ggml-amx/*.cpp")
48-
list(APPEND GGML_SOURCES_AMX "ggml-amx.cpp")
49-
endif()
50-
endif()
51-
52-
5332
if (GGML_HIPBLAS)
5433
if (NOT EXISTS $ENV{ROCM_PATH})
5534
if (NOT EXISTS /opt/rocm)
@@ -159,156 +138,6 @@ if (GGML_HIPBLAS)
159138
list(APPEND GGML_EXTRA_LIBS_PUBLIC hip::host roc::rocblas roc::hipblas)
160139
endif()
161140

162-
if (GGML_SYCL)
163-
if (NOT GGML_SYCL_TARGET MATCHES "^(INTEL|NVIDIA|AMD)$")
164-
message(FATAL_ERROR "Invalid backend chosen, supported options are INTEL, NVIDIA, or AMD")
165-
endif()
166-
167-
check_cxx_compiler_flag("-fsycl" SUPPORTS_SYCL)
168-
169-
if (DEFINED ENV{ONEAPI_ROOT})
170-
message(STATUS "Using oneAPI Release SYCL compiler (icpx).")
171-
elseif(SUPPORTS_SYCL)
172-
message(WARNING "Using open-source SYCL compiler (clang++). Didn't detect ENV {ONEAPI_ROOT}.
173-
If you expected the oneAPI Release compiler, please install oneAPI & source it, like:
174-
source /opt/intel/oneapi/setvars.sh")
175-
else()
176-
message(FATAL_ERROR, "C++ compiler lacks SYCL support.")
177-
endif()
178-
message(STATUS "SYCL found")
179-
#todo: AOT
180-
181-
list(APPEND GGML_CDEF_PUBLIC GGML_USE_SYCL)
182-
183-
if (GGML_SYCL_F16)
184-
if (GGML_SYCL_TARGET STREQUAL "AMD")
185-
message(WARNING "AMD target does not entirely support FP16 in the SYCL backend.")
186-
endif()
187-
add_compile_definitions(GGML_SYCL_F16)
188-
endif()
189-
190-
if (GGML_CUDA_FORCE_MMQ)
191-
add_compile_definitions(GGML_SYCL_FORCE_MMQ)
192-
endif()
193-
194-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing -fsycl")
195-
196-
if (GGML_SYCL_TARGET STREQUAL "NVIDIA")
197-
add_compile_definitions(GGML_SYCL_WARP_SIZE=32)
198-
elseif (GGML_SYCL_TARGET STREQUAL "AMD")
199-
# INFO: Allowed Sub_group_sizes are not consistent through all
200-
# hip targets. For example, 64 is used for certain models, but the backend
201-
# does not support it.
202-
# Target archs tested working: gfx1030, gfx1031, (Only tested sub_group_size = 32)
203-
add_compile_definitions(GGML_SYCL_WARP_SIZE=32)
204-
else()
205-
add_compile_definitions(GGML_SYCL_WARP_SIZE=16)
206-
endif()
207-
208-
file(GLOB GGML_HEADERS_SYCL "ggml-sycl/*.hpp")
209-
list(APPEND GGML_HEADERS_SYCL "../include/ggml-sycl.h")
210-
211-
file(GLOB GGML_SOURCES_SYCL "ggml-sycl/*.cpp")
212-
list(APPEND GGML_SOURCES_SYCL "ggml-sycl.cpp")
213-
214-
find_package(DNNL)
215-
message("-- DNNL found:" ${DNNL_FOUND})
216-
217-
if (GGML_SYCL_TARGET STREQUAL "INTEL")
218-
add_compile_definitions(GGML_SYCL_DNNL=${DNNL_FOUND})
219-
else()
220-
add_compile_definitions(GGML_SYCL_DNNL=0)
221-
endif()
222-
223-
if (${DNNL_FOUND} AND GGML_SYCL_TARGET STREQUAL "INTEL")
224-
list(APPEND GGML_EXTRA_LIBS_PRIVATE DNNL::dnnl)
225-
endif()
226-
227-
if (WIN32)
228-
find_package(IntelSYCL REQUIRED)
229-
find_package(MKL REQUIRED)
230-
list(APPEND GGML_EXTRA_LIBS_PRIVATE IntelSYCL::SYCL_CXX MKL::MKL MKL::MKL_SYCL)
231-
else()
232-
if (GGML_SYCL_TARGET STREQUAL "INTEL")
233-
list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread)
234-
elseif (GGML_SYCL_TARGET STREQUAL "NVIDIA")
235-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=nvptx64-nvidia-cuda")
236-
list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl pthread m dl onemkl)
237-
elseif (GGML_SYCL_TARGET STREQUAL "AMD")
238-
if (GGML_SYCL_HIP_TARGET STREQUAL "")
239-
message(ERROR "Can't enable SYCL hip backend, GGML_SYCL_HIP_TARGET has not been set.")
240-
endif()
241-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=${GGML_SYCL_HIP_TARGET}")
242-
list(APPEND GGML_EXTRA_LIBS_PRIVATE sycl pthread m dl onemkl)
243-
endif()
244-
endif()
245-
endif()
246-
247-
if (GGML_CANN)
248-
if ("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME})
249-
set(CANN_INSTALL_DIR $ENV{ASCEND_TOOLKIT_HOME})
250-
message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}")
251-
endif()
252-
253-
if (CANN_INSTALL_DIR)
254-
# Only Support Linux.
255-
if (GGML_CANN)
256-
if (NOT UNIX)
257-
set(GGML_CANN OFF)
258-
message(WARNING "CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}. Turning off GGML_CANN")
259-
endif()
260-
endif()
261-
262-
# Supported platforms: x86-64, arm64
263-
if (GGML_CANN)
264-
if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
265-
elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
266-
else()
267-
set(GGML_CANN OFF)
268-
message(WARNING "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off GGML_CANN")
269-
endif()
270-
endif()
271-
272-
# Set header and libs
273-
if(GGML_CANN)
274-
set(CANN_INCLUDE_DIRS
275-
${CANN_INSTALL_DIR}/include
276-
${CANN_INSTALL_DIR}/include/aclnn
277-
${CANN_INSTALL_DIR}/acllib/include
278-
)
279-
280-
add_subdirectory(ggml-cann/kernels)
281-
list(APPEND CANN_LIBRARIES
282-
ascendcl
283-
nnopbase
284-
opapi
285-
acl_op_compiler
286-
ascendc_kernels
287-
)
288-
289-
set(GGML_HEADERS_CANN "../include/ggml-cann.h")
290-
file(GLOB GGML_SOURCES_CANN "ggml-cann/*.cpp")
291-
list(APPEND GGML_SOURCES_CANN "ggml-cann.cpp")
292-
293-
message(STATUS "CANN: CANN_INCLUDE_DIRS = ${CANN_INCLUDE_DIRS}")
294-
message(STATUS "CANN: CANN_LIBRARIES = ${CANN_LIBRARIES}")
295-
296-
list(APPEND GGML_EXTRA_LIBS_PRIVATE ${CANN_LIBRARIES} )
297-
list(APPEND GGML_EXTRA_INCLUDES ${CANN_INCLUDE_DIRS})
298-
list(APPEND GGML_EXTRA_LIBDIRS ${CANN_INSTALL_DIR}/lib64)
299-
300-
list(APPEND GGML_CDEF_PUBLIC GGML_USE_CANN)
301-
endif()
302-
else()
303-
set(GGML_CANN OFF)
304-
message(WARNING "CANN: Can't find CANN_INSTALL_DIR, do you forget to source set_var.sh. Turning off GGML_CANN")
305-
endif()
306-
307-
if(NOT GGML_CANN)
308-
message(WARNING "CANN: GGML_CANN is turned OFF, see above for details.")
309-
endif()
310-
endif()
311-
312141
function(get_flags CCID CCVER)
313142
set(C_FLAGS "")
314143
set(CXX_FLAGS "")
@@ -485,15 +314,10 @@ if (WIN32)
485314
endif()
486315
endif()
487316

488-
#
489-
# libraries
490-
#
491-
492317
# ggml
493318

494319
add_library(ggml-base
495320
../include/ggml.h
496-
../include/ggml-alloc
497321
../include/ggml-alloc.h
498322
../include/ggml-backend.h
499323
../include/ggml-cpp.h
@@ -505,42 +329,43 @@ add_library(ggml-base
505329
ggml-quants.c
506330
ggml-quants.h
507331
ggml-aarch64.c
508-
ggml-aarch64.h
509-
)
332+
ggml-aarch64.h)
510333

511334
add_library(ggml
512-
ggml-backend-reg.cpp
513-
)
514-
add_subdirectory(ggml-cpu)
335+
ggml-backend-reg.cpp)
515336

516-
target_link_libraries(ggml PUBLIC ggml-base ggml-cpu)
337+
target_link_libraries(ggml PUBLIC ggml-base)
517338

518339
function(ggml_add_backend backend)
519340
string(TOUPPER "GGML_${backend}" backend_id)
520341
if (${backend_id})
521342
string(TOLOWER "ggml-${backend}" backend_target)
522343
add_subdirectory(${backend_target})
523-
if (${BUILD_SHARED_LIBS})
524-
target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
525-
target_compile_definitions(${backend_target} PUBLIC GGML_BACKEND_SHARED)
344+
if (${backend_id})
345+
message(STATUS "Including ${backend} backend")
346+
if (${BUILD_SHARED_LIBS})
347+
target_compile_definitions(${backend_target} PRIVATE GGML_BACKEND_BUILD)
348+
target_compile_definitions(${backend_target} PUBLIC GGML_BACKEND_SHARED)
349+
endif()
350+
target_link_libraries(ggml PUBLIC ${backend_target})
351+
string(TOUPPER "GGML_USE_${backend}" backend_use)
352+
target_compile_definitions(ggml PUBLIC ${backend_use})
526353
endif()
527-
target_link_libraries(ggml PUBLIC ${backend_target})
528-
string(TOUPPER "GGML_USE_${backend}" backend_use)
529-
#set(GGML_CDEF_PUBLIC ${GGML_CDEF_PUBLIC} ${backend_use} PARENT_SCOPE)
530-
target_compile_definitions(ggml PUBLIC ${backend_use})
531354
endif()
532355
endfunction()
533356

357+
set(GGML_CPU ON)
358+
ggml_add_backend(CPU)
359+
ggml_add_backend(AMX)
360+
ggml_add_backend(BLAS)
361+
ggml_add_backend(CANN)
534362
ggml_add_backend(CUDA)
363+
ggml_add_backend(Kompute)
535364
ggml_add_backend(METAL)
536-
ggml_add_backend(BLAS)
537365
ggml_add_backend(RPC)
366+
ggml_add_backend(SYCL)
538367
ggml_add_backend(Vulkan)
539368

540-
if (EMSCRIPTEN)
541-
set_target_properties(ggml PROPERTIES COMPILE_FLAGS "-msimd128")
542-
endif()
543-
544369
target_compile_definitions(ggml-base PUBLIC ${GGML_CDEF_PUBLIC})
545370
target_include_directories(ggml-base PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
546371
target_compile_features (ggml-base PRIVATE c_std_11) # don't bump
@@ -574,5 +399,7 @@ if (BUILD_SHARED_LIBS)
574399
set_target_properties(ggml-base PROPERTIES POSITION_INDEPENDENT_CODE ON)
575400
set_target_properties(ggml PROPERTIES POSITION_INDEPENDENT_CODE ON)
576401
target_compile_definitions(ggml-base PRIVATE GGML_BUILD)
402+
target_compile_definitions(ggml PRIVATE GGML_BUILD)
577403
target_compile_definitions(ggml-base PUBLIC GGML_SHARED)
404+
target_compile_definitions(ggml PUBLIC GGML_SHARED)
578405
endif()

‎ggml/src/ggml-amx/CMakeLists.txt‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.0)
2+
else()
3+
set(GGML_AMX OFF PARENT_SCOPE)
4+
message(WARNING "AMX requires gcc version > 11.0. Turning off GGML_AMX.")
5+
endif()
6+
7+
if (GGML_AMX)
8+
message(STATUS "Using AMX")
9+
10+
file(GLOB GGML_HEADERS_AMX "*.h")
11+
list(APPEND GGML_HEADERS_AMX "../../include/ggml-amx.h")
12+
13+
file(GLOB GGML_SOURCES_AMX "*.cpp")
14+
15+
add_library(ggml-amx
16+
${GGML_HEADERS_AMX}
17+
${GGML_SOURCES_AMX})
18+
19+
target_link_libraries(ggml-amx PRIVATE ggml-base)
20+
target_include_directories(ggml-amx PRIVATE . ..)
21+
22+
# this is duplicated from the CPU backend, since the AMX backend also depends on the architecture flags
23+
# TODO: integrate AMX backend into the CPU backend
24+
if (MSVC)
25+
# instruction set detection for MSVC only
26+
if (GGML_NATIVE)
27+
# TODO: improve, should not reference files from the parent folder
28+
include(../ggml-cpu/cmake/FindSIMD.cmake)
29+
endif ()
30+
if (GGML_AVX512)
31+
list(APPEND ARCH_FLAGS /arch:AVX512)
32+
# MSVC has no compile-time flags enabling specific
33+
# AVX512 extensions, neither it defines the
34+
# macros corresponding to the extensions.
35+
# Do it manually.
36+
if (GGML_AVX512_VBMI)
37+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VBMI__>)
38+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VBMI__>)
39+
endif()
40+
if (GGML_AVX512_VNNI)
41+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512VNNI__>)
42+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512VNNI__>)
43+
endif()
44+
if (GGML_AVX512_BF16)
45+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AVX512BF16__>)
46+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AVX512BF16__>)
47+
endif()
48+
if (GGML_AMX_TILE)
49+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_TILE__>)
50+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_TILE__>)
51+
endif()
52+
if (GGML_AMX_INT8)
53+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_INT8__>)
54+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_INT8__>)
55+
endif()
56+
if (GGML_AMX_BF16)
57+
add_compile_definitions($<$<COMPILE_LANGUAGE:C>:__AMX_BF16__>)
58+
add_compile_definitions($<$<COMPILE_LANGUAGE:CXX>:__AMX_BF16__>)
59+
endif()
60+
elseif (GGML_AVX2)
61+
list(APPEND ARCH_FLAGS /arch:AVX2)
62+
elseif (GGML_AVX)
63+
list(APPEND ARCH_FLAGS /arch:AVX)
64+
endif()
65+
else()
66+
if (GGML_NATIVE)
67+
list(APPEND ARCH_FLAGS -march=native)
68+
endif()
69+
if (GGML_F16C)
70+
list(APPEND ARCH_FLAGS -mf16c)
71+
endif()
72+
if (GGML_FMA)
73+
list(APPEND ARCH_FLAGS -mfma)
74+
endif()
75+
if (GGML_AVX)
76+
list(APPEND ARCH_FLAGS -mavx)
77+
endif()
78+
if (GGML_AVX2)
79+
list(APPEND ARCH_FLAGS -mavx2)
80+
endif()
81+
if (GGML_AVX512)
82+
list(APPEND ARCH_FLAGS -mavx512f)
83+
list(APPEND ARCH_FLAGS -mavx512dq)
84+
list(APPEND ARCH_FLAGS -mavx512bw)
85+
endif()
86+
if (GGML_AVX512_VBMI)
87+
list(APPEND ARCH_FLAGS -mavx512vbmi)
88+
endif()
89+
if (GGML_AVX512_VNNI)
90+
list(APPEND ARCH_FLAGS -mavx512vnni)
91+
endif()
92+
if (GGML_AVX512_BF16)
93+
list(APPEND ARCH_FLAGS -mavx512bf16)
94+
endif()
95+
if (GGML_AMX_TILE)
96+
list(APPEND ARCH_FLAGS -mamx-tile)
97+
endif()
98+
if (GGML_AMX_INT8)
99+
list(APPEND ARCH_FLAGS -mamx-int8)
100+
endif()
101+
if (GGML_AMX_BF16)
102+
list(APPEND ARCH_FLAGS -mamx-bf16)
103+
endif()
104+
endif()
105+
106+
target_compile_options(ggml-amx PRIVATE ${ARCH_FLAGS})
107+
endif()

‎ggml/src/ggml-amx/common.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
#pragma once
22

33
#include "ggml.h"
4-
#include "ggml-cpu-impl.h" // <immintrin.h>
4+
// hack until AMX is moved into the CPU backend
5+
#include "../ggml-cpu/ggml-cpu-impl.h" // <immintrin.h>
56

67
#include <algorithm>
78
#include <memory>

0 commit comments

Comments
 (0)