Integrate copy2fpga filetransfer

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "kompute"]
 	path = ggml/src/ggml-kompute/kompute
 	url = https://github.com/nomic-ai/kompute.git
+[submodule "ggml-tsi-kernel"]
+	path = ggml-tsi-kernel
+	url = [email protected]:tsisw/ggml-tsi-kernel.git

CMakeLists.txt

@@ -5,10 +5,61 @@ include(CheckIncludeFileCXX)
 #set(CMAKE_WARN_DEPRECATED YES)
 set(CMAKE_WARN_UNUSED_CLI YES)
 
+if (GGML_TSAVORITE)
+    if  (NOT DEFINED GGML_TSAVORITE_TARGET)
+        set(GGML_TSAVORITE_TARGET "posix")
+    endif()
+    if (NOT ${GGML_TSAVORITE_TARGET} STREQUAL fpga)
+        set(GGML_TSAVORITE_TARGET "posix")
+    endif()
+
+    if (NOT DEFINED MLIR_COMPILER_DIR)
+        if (NOT DEFINED $ENV{MLIR_SDK_VERSION})
+            set (MLIR_COMPILER_DIR /proj/rel/sw/sdk-r.0.1.3/compiler)
+            message("MLIR_SDK_VERSION not set defaulting to ${MLIR_COMPILER_DIR}")
+        else()
+            set (MLIR_COMPILER_DIR $ENV{MLIR_SDK_VERSION}/compiler)
+        endif()
+    endif()
+
+    if (NOT DEFINED RUNTIME_DIR)
+        if (NOT DEFINED $ENV{MLIR_SDK_VERSION})
+            set (RUNTIME_DIR /proj/rel/sw/sdk-r.0.1.3/${GGML_TSAVORITE_TARGET}/runtime)
+            message("MLIR_SDK_VERSION not set defaulting to ${RUNTIME_DIR}")
+        else()
+            set (RUNTIME_DIR $ENV{MLIR_SDK_VERSION}/${GGML_TSAVORITE_TARGET}/runtime)
+        endif()
+    endif()
+
+    if (NOT DEFINED GGML_TSI_KERNEL_DIR)
+       set (GGML_TSI_KERNEL_DIR ${CMAKE_SOURCE_DIR}/ggml-tsi-kernel/${GGML_TSAVORITE_TARGET})
+    endif()
+
+
+    if (${GGML_TSAVORITE_TARGET} STREQUAL fpga)
+        set(CMAKE_CROSSCOMPILING ON)
+        set(ARCH_FLAGS -march=armv8-a)
+        file(GLOB TLIBS  "${RUNTIME_DIR}/lib/*.so" "${GGML_TSI_KERNEL_DIR}/host/*.o" "${RUNTIME_DIR}/../utils/lib/TsavRTShimCAPI.cpp.o")
+        message("Setting target as fpga")
+    elseif (${GGML_TSAVORITE_TARGET} STREQUAL "posix")
+        file(GLOB TLIBS  "${RUNTIME_DIR}/lib/*.so" "${GGML_TSI_KERNEL_DIR}/host/*.o" "${MLIR_COMPILER_DIR}/lib/libFFMDeviceShim.so" "${MLIR_COMPILER_DIR}/lib/libTsavRTPosixShimCAPI.so")
+        message("Setting target as posix for tsavorite")
+    endif()
+
+    set(GGML_TSAVORITE_TARGET "${GGML_TSAVORITE_TARGET}" CACHE STRING "Target for tsavorite")
+    set (TSAVORITE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/ggml/src/ggml-tsavorite/include)
+
+    include_directories(${TSAVORITE_INCLUDE_DIR})
+    include_directories(${MLIR_COMPILER_DIR}/include/runtime/shim)
+    include_directories(${RUNTIME_DIR}/include)
+    message("tsavorite backend is enabled")
+endif()
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
 if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    #set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
+    set(CMAKE_BUILD_TYPE Debug CACHE STRING "Build type" FORCE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
 endif()
 
@@ -82,9 +133,18 @@ option(LLAMA_BUILD_EXAMPLES "llama: build examples"       ${LLAMA_STANDALONE})
 option(LLAMA_BUILD_SERVER   "llama: build server example" ${LLAMA_STANDALONE})
 
 # 3rd party libs
-option(LLAMA_CURL       "llama: use libcurl to download model from an URL" ON)
 option(LLAMA_LLGUIDANCE "llama-common: include LLGuidance library for structured output in common utils" OFF)
 
+if (GGML_TSAVORITE)
+    if (${GGML_TSAVORITE_TARGET} STREQUAL fpga)
+        option(LLAMA_CURL       "llama: use libcurl to download model from an URL" OFF)
+    else()
+        option(LLAMA_CURL       "llama: use libcurl to download model from an URL" ON)
+    endif()
+else()
+    option(LLAMA_CURL       "llama: use libcurl to download model from an URL" ON)
+endif()
+
 # Required for relocatable CMake package
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/common.cmake)

README.md

@@ -580,3 +580,59 @@ $ echo "source ~/.llama-completion.bash" >> ~/.bashrc
 - [minja](https://github.com/google/minja) - Minimal Jinja parser in C++, used by various tools/examples - MIT License
 - [linenoise.cpp](./tools/run/linenoise.cpp/linenoise.cpp) - C++ library that provides readline-like line editing capabilities, used by `llama-run` - BSD 2-Clause License
 - [curl](https://curl.se/) - Client-side URL transfer library, used by various tools/examples - [CURL License](https://curl.se/docs/copyright.html)
+
+#### TSI compilation steps
+```bash
+#Pull the repo frim tsisw as follows
+git clone [email protected]:tsisw/llama.cpp.git
+
+#Ensure prerequisites are met as follows
+cd llama.cpp/
+#Ensure prerequisites are met as follows
+echo 'updating submodule'
+git submodule update --recursive --init
+cd ggml-tsi-kernel/
+module load tsi4 gcc/13.3.0
+export MLIR_SDK_VERSION=/proj/rel/sw/sdk-r.0.1.3
+echo 'creating python virtual env'
+/proj/local/Python-3.10.12/bin/python3 -m venv blob-creation
+source blob-creation/bin/activate
+echo 'installing mlir and python dependencies'
+pip install -r ${MLIR_SDK_VERSION}/compiler/python/requirements-common.txt
+pip install ${MLIR_SDK_VERSION}/compiler/python/mlir_external_packages-1.3.0-py3-none-any.whl
+pip install onnxruntime-training
+
+
+
+#build TSI kernels for the Tsavorite backend
+#First for FPGA
+cd fpga-kernel
+cmake -B build-fpga
+./create-all-kernels.sh
+#The for Posix Use cases
+cd ../posix-kernel/
+./create-all-kernels.sh
+
+#Change directory to top level llama.cpp
+cd ../../
+
+#Compile for posix with build-posix as a target folder
+
+cmake -B build-posix -DGGML_TSAVORITE=ON -DGGML_TSAVORITE_TARGET=posix
+cmake --build build-posix --config Release
+
+#Compile for fpga with build-fpga as a target folder
+export CC="/proj/rel/sw/arm-gnu-toolchain-14.2.rel1-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc"
+export CXX="/proj/rel/sw/arm-gnu-toolchain-14.2.rel1-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-g++"
+cmake -B build-fpga -DGGML_TSAVORITE=ON -DGGML_TSAVORITE_TARGET=fpga
+cmake --build build-fpga --config Release
+
+#For easy build one can also use which creates a FPGA specific tar bundle tsi-ggml.tz
+#If you want to release the build update the TSI-VERSION in the file tsi-pkg-build.sh and add Release as parameter
+#when running ./tsi-pkg-build.sh (Note it will overwrite what exists in /proj/rel/sw/ggml so be sure you want to do
+#it. Example ./tsi-pkg-build.sh release
+./tsi-pkg-build.sh
+
+```
+
+## References

common/CMakeLists.txt

@@ -145,8 +145,16 @@ endif ()
 
 target_include_directories(${TARGET} PUBLIC .)
 target_compile_features   (${TARGET} PUBLIC cxx_std_17)
-target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
 
+if (GGML_TSAVORITE)
+    if (${GGML_TSAVORITE_TARGET} STREQUAL fpga)
+        target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} ${TLIBS} PUBLIC llama Threads::Threads)
+    else()
+        target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
+    endif()
+else()
+    target_link_libraries     (${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama Threads::Threads)
+endif()
 
 #
 # copy the license files

common/log.h

@@ -90,11 +90,20 @@ void common_log_set_timestamps(struct common_log * log,       bool   timestamps)
 #define LOG(...)             LOG_TMPL(GGML_LOG_LEVEL_NONE, 0,         __VA_ARGS__)
 #define LOGV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_NONE, verbosity, __VA_ARGS__)
 
+#if ENABLE_LOG
 #define LOG_INF(...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  0,                 __VA_ARGS__)
 #define LOG_WRN(...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  0,                 __VA_ARGS__)
 #define LOG_ERR(...) LOG_TMPL(GGML_LOG_LEVEL_ERROR, 0,                 __VA_ARGS__)
 #define LOG_DBG(...) LOG_TMPL(GGML_LOG_LEVEL_DEBUG, LOG_DEFAULT_DEBUG, __VA_ARGS__)
 #define LOG_CNT(...) LOG_TMPL(GGML_LOG_LEVEL_CONT,  0,                 __VA_ARGS__)
+#else
+#define LOG_INF(...)
+#define LOG_WRN(...)
+#define LOG_ERR(...)
+#define LOG_DBG(...)
+#define LOG_CNT(...)
+#endif
+#define LOG_TSAVORITE(...) LOG_TMPL(GGML_LOG_LEVEL_TSAVORITE,  0,                 __VA_ARGS__)
 
 #define LOG_INFV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_INFO,  verbosity, __VA_ARGS__)
 #define LOG_WRNV(verbosity, ...) LOG_TMPL(GGML_LOG_LEVEL_WARN,  verbosity, __VA_ARGS__)

docs/build.md

@@ -559,3 +559,66 @@ The GPU may still be used to accelerate some parts of the computation even when
 In most cases, it is possible to build and use multiple backends at the same time. For example, you can build llama.cpp with both CUDA and Vulkan support by using the `-DGGML_CUDA=ON -DGGML_VULKAN=ON` options with CMake. At runtime, you can specify which backend devices to use with the `--device` option. To see a list of available devices, use the `--list-devices` option.
 
 Backends can be built as dynamic libraries that can be loaded dynamically at runtime. This allows you to use the same llama.cpp binary on different machines with different GPUs. To enable this feature, use the `GGML_BACKEND_DL` option when building.
+
+
+## TSI compilation steps
+
+Following are the instructions to compile for TSI FPGA and Posix backend
+
+```bash
+Pull the repo frim tsisw as follows
+git clone [email protected]:tsisw/llama.cpp.git -b FIR-699
+```
+
+Ensure prerequisites are met as follows
+```bash
+cd llama.cpp/
+git submodule update --recursive --init
+cd ggml-tsi-kernel/
+module load tsi4 gcc/13.3.0
+python3 -m venv blob-creation
+source blob-creation/bin/activate
+pip install -r /proj/rel/sw/sdk-r.0.1.3/compiler/python/requirements-common.txt
+pip install /proj/rel/sw/sdk-r.0.1.3/compiler/python/mlir_external_packages-1.3.0-py3-none-any.whl
+pip install onnxruntime-training
+```
+
+build TSI kernels for the Tsavorite backend
+First for FPGA
+```bash
+cd fpga-kernel
+cmake -B build-fpga
+./create-all-kernels.sh
+```
+The for Posix Use cases
+```bash
+cd ../posix-kernel/
+./create-all-kernels.sh
+```
+
+Change directory to top level llama.cpp
+```bash
+cd ../../
+```
+
+Compile for posix with build-posix as a target folder
+```bash
+cmake -B build-posix -DGGML_TSAVORITE=ON -DGGML_TSAVORITE_TARGET=posix
+cmake --build build-posix --config Release
+```
+
+Compile for fpga with build-fpga as a target folder
+```bash
+export CC="/proj/rel/sw/arm-gnu-toolchain-14.2.rel1-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-gcc"
+export CXX="/proj/rel/sw/arm-gnu-toolchain-14.2.rel1-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu-g++"
+cmake -B build-fpga -DGGML_TSAVORITE=ON -DGGML_TSAVORITE_TARGET=fpga
+cmake --build build-fpga --config Release
+```
+For easy build one can also use which creates a FPGA specific tar bundle tsi-ggml.tz
+If you want to release the build update the TSI-VERSION in the file tsi-pkg-build.sh and add Release as parameter
+when running ./tsi-pkg-build.sh (Note it will overwrite what exists in /proj/rel/sw/ggml so be sure you want to do
+it. Example ./tsi-pkg-build.sh release
+
+```bash
+./tsi-pkg-build.sh
+```

examples/gguf-hash/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(TARGET llama-gguf-hash)
 add_executable(${TARGET} gguf-hash.cpp)
+target_link_libraries(${TARGET} PRIVATE ${TLIBS})
 install(TARGETS ${TARGET} RUNTIME)
 
 # clibs dependencies

examples/gguf/CMakeLists.txt

@@ -1,5 +1,5 @@
 set(TARGET llama-gguf)
 add_executable(${TARGET} gguf.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/lookup/CMakeLists.txt

@@ -1,23 +1,23 @@
 set(TARGET llama-lookup)
 add_executable(${TARGET} lookup.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
 
 set(TARGET llama-lookup-create)
 add_executable(${TARGET} lookup-create.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
 
 set(TARGET llama-lookup-merge)
 add_executable(${TARGET} lookup-merge.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
 
 set(TARGET llama-lookup-stats)
 add_executable(${TARGET} lookup-stats.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/simple-chat/CMakeLists.txt

@@ -1,5 +1,5 @@
 set(TARGET llama-simple-chat)
 add_executable(${TARGET} simple-chat.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT} ${TLIBS})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)

examples/simple/CMakeLists.txt

@@ -1,5 +1,24 @@
+#
+# simple-ctx
 set(TARGET llama-simple)
 add_executable(${TARGET} simple.cpp)
 install(TARGETS ${TARGET} RUNTIME)
-target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(${TARGET} PRIVATE llama ${TLIBS} ${CMAKE_THREAD_LIBS_INIT})
 target_compile_features(${TARGET} PRIVATE cxx_std_17)
+
+#
+if (GGML_TSAVORITE)
+    #
+    # tsavorite backend test cases
+    if (${GGML_TSAVORITE_TARGET} STREQUAL fpga)
+        file(GLOB TLIBS  "${RUNTIME_DIR}/lib/*.so" "../../${GGML_TSI_KERNEL_DIR}/host/*.o")
+    else()
+        file(GLOB TLIBS  "${RUNTIME_DIR}/lib/*.so" "${MLIR_COMPILER_DIR}/lib/libFFMDeviceShim.so" "../../${GGML_TSI_KERNEL_DIR}/host/*.o")
+    endif()
+    #
+    # simple-backend-tsi
+
+    set(TEST_TARGET simple-backend-tsi)
+    add_executable(${TEST_TARGET} simple-backend-tsi.cpp)
+    target_link_libraries(${TEST_TARGET} PRIVATE ggml ${TLIBS}  dl rt)
+endif()