[GCU] Add gcu kernels (#1410)

Author: EnflameGCU

backends/gcu/CMakeLists.txt

@@ -17,39 +17,56 @@ cmake_minimum_required(VERSION 3.10)
 project(paddle-custom-gcu CXX C)
 set(CUSTOM_GCU_NAME "paddle-custom-gcu")
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 message(STATUS "CMAKE_DEBUG CMAKE_MODULE_PATH is: ${CMAKE_MODULE_PATH}")
+option(VERSION_WITH_GIT "Use git hash to PACKAGE_VERSION" FALSE)
 
 set(TOPS_RELEASE_VERSION 3.1.0)
 if((NOT DEFINED PACKAGE_VERSION)
    OR ("${PACKAGE_VERSION}" STREQUAL "")
    OR ("${PACKAGE_VERSION}" STREQUAL "123.456"))
-  # get Paddle-custom git hash
-  execute_process(
-    COMMAND git log -1 --abbrev=7 --format=%h
-    WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-    OUTPUT_VARIABLE GIT_HASH
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-  message(STATUS "git hash: ${GIT_HASH}")
-  set(GIT_HASH ".${GIT_HASH}")
-else()
-  set(GIT_HASH "")
-  set(TOPS_RELEASE_VERSION ${PACKAGE_VERSION})
+  if(VERSION_WITH_GIT)
+    # get Paddle-custom git hash
+    execute_process(
+      COMMAND git log -1 --abbrev=7 --format=%h
+      WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
+      OUTPUT_VARIABLE _tag
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    message(STATUS "git hash: ${_tag}")
+  else()
+    string(TIMESTAMP _tag "%Y%m%d")
+  endif()
+  set(PACKAGE_VERSION ${TOPS_RELEASE_VERSION}.${_tag})
 endif()
-set(PACKAGE_VERSION ${GIT_HASH}-${TOPS_RELEASE_VERSION})
 
-message(STATUS "tops release version: ${TOPS_RELEASE_VERSION}")
 message(STATUS "package version: ${PACKAGE_VERSION}")
 
+if(DEFINED PY_VERSION)
+  message(STATUS "User define PY_VERSION: ${PY_VERSION}")
+else()
+  set(PY_VERSION "3.10")
+  message(STATUS "Use default PY_VERSION: ${PY_VERSION}")
+endif()
+set(PYTHON_VERSION ${PY_VERSION})
+set(Python_EXECUTABLE "python${PY_VERSION}")
+message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
+
+if(NOT DEFINED ENV{PADDLE_VERSION})
+  set(ENV{PADDLE_VERSION} 3.0.0-beta1)
+endif()
+
 include(paddle)
 include(version)
 include(generic)
 include(external/gcu)
+include(external/topscc)
 
-include_directories(${CMAKE_SOURCE_DIR})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(/opt/tops/include)
 
-set(OUTPUT_PADDLE_PACKAGE_VERSION ${PADDLE_VERSION}${PACKAGE_VERSION})
+set(OUTPUT_PADDLE_PACKAGE_VERSION ${PADDLE_VERSION}+${PACKAGE_VERSION})
+string(REPLACE "-beta" "b" OUTPUT_PADDLE_PACKAGE_VERSION
+               "${OUTPUT_PADDLE_PACKAGE_VERSION}")
 option(WITH_KERNELS "compile with custom kernels" ON)
 option(WITH_TESTING "compile with unit testing" OFF)
 option(WITH_MKL "compile with mkl support" ON)
@@ -62,12 +79,6 @@ message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: "
                "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
 message(STATUS "AR tools: ${CMAKE_AR}")
 
-if(DEFINED PY_VERSION)
-  message(STATUS "User define PY_VERSION: ${PY_VERSION}")
-  set(Python_EXECUTABLE "python${PY_VERSION}")
-endif()
-message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
-
 # custom runtime
 set(CUSTOM_GCU_SRCS runtime/runtime.cc)
 add_definitions(-DPADDLE_WITH_CUSTOM_DEVICE)
@@ -81,12 +92,12 @@ if(WITH_KERNELS)
   add_compile_definitions(HLIR_BUILDER_ABI_COMPATIABLE)
   file(
     GLOB_RECURSE CUSTOM_KERNEL_SRCS
-    RELATIVE ${CMAKE_SOURCE_DIR}
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
     kernels/*.cc)
   list(APPEND CUSTOM_GCU_SRCS ${CUSTOM_KERNEL_SRCS})
   file(
     GLOB_RECURSE GCU_KERNEL_DEPENDS
-    RELATIVE ${CMAKE_SOURCE_DIR}
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
     backend/*.cc common/*.cc)
   # build static library
   set(GCU_DEPENDS_NAME "gcu-kernels-depend")
@@ -99,7 +110,7 @@ if(WITH_KERNELS)
   # custom op with kernel
   file(
     GLOB_RECURSE CUSTOM_CUSTOM_OP_SRCS
-    RELATIVE ${CMAKE_SOURCE_DIR}
+    RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
     custom_op/*.cc)
   list(APPEND CUSTOM_GCU_SRCS ${CUSTOM_CUSTOM_OP_SRCS})
 endif()
@@ -108,22 +119,44 @@ if(ON_INFER)
   link_directories(${PADDLE_INFERENCE_LIB_DIR})
 endif()
 
+# topscc kernels
+file(GLOB_RECURSE TOPSCC_KERNEL_SRCS
+     ${CMAKE_CURRENT_SOURCE_DIR}/topscc_custom_kernels/*.cc
+     ${CMAKE_CURRENT_SOURCE_DIR}/topscc_custom_kernels/*.cpp)
+list(LENGTH TOPSCC_KERNEL_SRCS TOPSCC_KERNEL_SRCS_LEN)
+unset(TOPSCC_LIBS)
+if(${TOPSCC_KERNEL_SRCS_LEN} GREATER 0)
+  topscc_compile(FALSE ${TOPSCC_KERNEL_SRCS} "-shared -fPIC -O3 " TOPSCC_LIBS)
+  message(STATUS "TOPSCC_LIBS: ${TOPSCC_LIBS}")
+endif()
+
 # build shared library
 add_library(${CUSTOM_GCU_NAME} SHARED ${CUSTOM_GCU_SRCS})
+
+# for link topscc custom kernels
+set_target_properties(
+  ${CUSTOM_GCU_NAME}
+  PROPERTIES
+    LINK_FLAGS
+    "-Wl,-rpath,/usr/local/lib/python${PY_VERSION}/dist-packages/paddle_custom_device/gcu/"
+)
+
 target_link_libraries(${CUSTOM_GCU_NAME} PRIVATE ${GCU_LIBS})
 if(WITH_KERNELS)
   target_link_libraries(${CUSTOM_GCU_NAME} PRIVATE ${GCU_DEPENDS_NAME})
-  target_link_libraries(
-    ${CUSTOM_GCU_NAME}
-    PRIVATE -Wl,--whole-archive ${TOPSOP_LIB_DIR}/libtopsop.a
-            -Wl,--no-whole-archive)
 endif()
 if(TOPS_MODULE_SOURCE_DIR)
   target_include_directories(${CUSTOM_GCU_NAME}
                              PRIVATE ${TOPS_MODULE_SOURCE_DIR}/usr/include)
   target_include_directories(${CUSTOM_GCU_NAME}
                              PRIVATE ${TOPS_MODULE_SOURCE_DIR}/opt/tops/include)
 endif()
+# link topscc kernel libs
+add_custom_target(CUSTOM_TOPSCC_LIBS ALL DEPENDS ${TOPSCC_LIBS})
+if(${TOPSCC_KERNEL_SRCS_LEN} GREATER 0)
+  add_dependencies(${CUSTOM_GCU_NAME} CUSTOM_TOPSCC_LIBS)
+  target_link_libraries(${CUSTOM_GCU_NAME} PRIVATE ${TOPSCC_LIBS})
+endif()
 
 # link paddle shared library
 include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/third_party.cmake)
@@ -142,11 +175,12 @@ else()
 endif()
 
 # export paddle_gcu interface with version map
-set_property(TARGET ${CUSTOM_GCU_NAME}
-             PROPERTY LINK_DEPENDS ${CMAKE_SOURCE_DIR}/paddle_gcu_export.map)
+set_property(
+  TARGET ${CUSTOM_GCU_NAME}
+  PROPERTY LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/paddle_gcu_export.map)
 target_link_options(
   ${CUSTOM_GCU_NAME} PRIVATE
-  -Wl,--version-script=${CMAKE_SOURCE_DIR}/paddle_gcu_export.map)
+  -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/paddle_gcu_export.map)
 
 # testing
 if(WITH_TESTING)
@@ -155,7 +189,7 @@ if(WITH_TESTING)
   add_subdirectory(tests)
   add_custom_command(
     OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tests/.timestamp
-    COMMAND cp -r ${CMAKE_SOURCE_DIR}/tests ${CMAKE_CURRENT_BINARY_DIR})
+    COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/tests ${CMAKE_CURRENT_BINARY_DIR})
   add_custom_target(python_tests ALL
                     DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/tests/.timestamp)
 endif()
@@ -194,13 +228,35 @@ add_custom_target(
     ${CMAKE_CURRENT_BINARY_DIR}/python/paddle_custom_device/lib${CUSTOM_GCU_NAME}.so
 )
 
+set(topscc_kernel_lib_targets "")
+unset(topscc_kernel_lib_targets)
+foreach(topscc_kernel_lib ${TOPSCC_LIBS})
+  get_filename_component(topscc_kernel_lib_name ${topscc_kernel_lib} NAME_WLE)
+  get_filename_component(topscc_kernel_lib_target_name ${topscc_kernel_lib}
+                         NAME_WE)
+  message(STATUS "topscc_kernel_lib_name: ${topscc_kernel_lib_name}")
+  add_custom_command(
+    OUTPUT
+      ${CMAKE_CURRENT_BINARY_DIR}/python/paddle_custom_device/gcu/${topscc_kernel_lib_name}
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${topscc_kernel_lib}
+            ${CMAKE_CURRENT_BINARY_DIR}/python/paddle_custom_device/gcu
+    USES_TERMINAL
+    DEPENDS ${topscc_kernel_lib})
+  list(
+    APPEND
+    topscc_kernel_lib_targets
+    ${CMAKE_CURRENT_BINARY_DIR}/python/paddle_custom_device/gcu/${topscc_kernel_lib_name}
+  )
+endforeach()
+add_custom_target(topscc_targets ALL DEPENDS ${topscc_kernel_lib_targets})
+
 set(whl_file
-    "${CMAKE_BINARY_DIR}/dist/paddle_custom_gcu-${OUTPUT_PADDLE_PACKAGE_VERSION}-cp${PY_VERSION_NO_DOT}-cp${PY_VERSION_NO_DOT}-linux_x86_64.whl"
+    "${CMAKE_CURRENT_BINARY_DIR}/dist/paddle_custom_gcu-${OUTPUT_PADDLE_PACKAGE_VERSION}-cp${PY_VERSION_NO_DOT}-cp${PY_VERSION_NO_DOT}-linux_x86_64.whl"
 )
 set(_passes_target_dir
     "${CMAKE_CURRENT_BINARY_DIR}/python/paddle_custom_device/gcu/passes")
 file(MAKE_DIRECTORY ${_passes_target_dir})
-file(GLOB passes_srcs "${CMAKE_SOURCE_DIR}/passes/*")
+file(GLOB passes_srcs "${CMAKE_CURRENT_SOURCE_DIR}/passes/*")
 foreach(passes_src IN LISTS passes_srcs)
   get_filename_component(passes_file_name ${passes_src} NAME)
   add_custom_command(
@@ -212,11 +268,11 @@ foreach(passes_src IN LISTS passes_srcs)
 endforeach()
 add_custom_command(
   OUTPUT ${whl_file}
-  COMMAND ${Python_EXECUTABLE} ${CMAKE_BINARY_DIR}/setup.py bdist_wheel
+  COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/setup.py bdist_wheel
   # COMMAND rename 's/_${TOPS_RELEASE_VERSION}/-${TOPS_RELEASE_VERSION}/g'
   # ${CMAKE_CURRENT_BINARY_DIR}/dist/*.whl
-  DEPENDS ${CUSTOM_GCU_NAME} CUSTOM_GCU_NAME_shared_lib_copy
-          ${CMAKE_BINARY_DIR}/setup.py ${passes_bin_files_list}
+  DEPENDS ${CUSTOM_GCU_NAME} CUSTOM_GCU_NAME_shared_lib_copy topscc_targets
+          ${CMAKE_CURRENT_BINARY_DIR}/setup.py ${passes_bin_files_list}
   COMMENT "Packing whl packages------>>>")
 
 add_custom_target(python_package ALL DEPENDS ${whl_file})

backends/gcu/backend/executor/cast_runner.h

@@ -13,11 +13,12 @@
 // limitations under the License.
 
 #pragma once
+#include <tops/tops_ext.h>
+
 #include <vector>
 
 #include "backend/executor/gcu_node.h"
 #include "backend/utils/utils.h"
-#include "tops/tops_ext.h"
 
 namespace backend {
 

backends/gcu/cmake/external/gcu.cmake

@@ -17,20 +17,14 @@ set(TOPS_INCLUDE_DIR "/usr/include/tops")
 set(TOPSTX_INCLUDE_DIR "/usr/include/topstx")
 set(GCU_LIB_DIR "/usr/lib")
 set(TOPS_LIB_DIR "/opt/tops/lib")
-set(OP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernels/topsflame/include/")
-set(TOPSOP_LIB_DIR "${CMAKE_CURRENT_SOURCE_DIR}/kernels/topsflame/")
 
 set(TOPS_VERSION "???")
 
 include_directories(${GCU_INCLUDE_DIR} ${TOPS_INCLUDE_DIR}
-                    ${TOPSTX_INCLUDE_DIR} ${OP_INCLUDE_DIR})
+                    ${TOPSTX_INCLUDE_DIR})
 
 find_library(SDK_LIB NAMES dtu_sdk ${GCU_LIB_DIR})
 find_library(ECCL_LIB NAMES eccl ${GCU_LIB_DIR})
-find_library(
-  RUNTIME_LIB
-  NAMES efrt ${GCU_LIB_DIR}
-  HINTS ${TOPS_LIB_DIR})
 find_library(
   TOPS_RT_LIB
   NAMES topsrt ${GCU_LIB_DIR}
@@ -40,9 +34,17 @@ find_library(
   NAMES topstx ${GCU_LIB_DIR}
   HINTS ${TOPS_LIB_DIR})
 find_library(TOPSATENOP_LIB NAMES topsaten ${GCU_LIB_DIR})
+find_library(TOPSCL_LIB NAMES topscl ${GCU_LIB_DIR})
 
-set(GCU_LIBS ${SDK_LIB} ${ECCL_LIB} ${RUNTIME_LIB} ${TOPS_RT_LIB} ${TOPSTX_LIB}
-             ${TOPSATENOP_LIB})
+set(GCU_LIBS
+    -Wl,--no-as-needed
+    ${SDK_LIB}
+    ${ECCL_LIB}
+    ${TOPS_RT_LIB}
+    ${TOPSTX_LIB}
+    ${TOPSATENOP_LIB}
+    ${TOPSCL_LIB}
+    -Wl,--as-needed)
 
 set(VERSION_REGEX "( [0-9]+\.[0-9]+\.(RC)?[0-9]*) ")
 macro(find_gcu_version component)

backends/gcu/cmake/external/topscc.cmake

@@ -0,0 +1,51 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+if(NOT EXISTS /usr/bin/topscc)
+  message(FATAL_ERROR "TOPSCC not found")
+else()
+  set(TOPSCC /usr/bin/topscc)
+endif()
+
+function(topscc_compile is_static_lib topscc_files compile_options out_libs)
+  message(STATUS "== Start compiling topscc kernels ==")
+  message(STATUS "[KERNEL FILES]: ${topscc_files}")
+  message(STATUS "[COMPILE OPTIONS]: ${compile_options}")
+  message(STATUS "[IS STATIC LIB]: ${is_static_lib}")
+
+  set(outs)
+  # string(CONCAT compile_options ${compile_options} "-ltops")
+  foreach(file ${topscc_files})
+    message(STATUS "[TOPSCC FILE]: ${file}")
+    get_filename_component(file_name ${file} NAME_WE)
+    message(STATUS "[TOPSCC Name]: ${file_name}")
+
+    if(is_static_lib)
+      set(out ${CMAKE_CURRENT_BINARY_DIR}/lib${file_name}.a)
+    else()
+      set(out ${CMAKE_CURRENT_BINARY_DIR}/lib${file_name}.so)
+    endif()
+    add_custom_command(
+      OUTPUT ${out}
+      COMMAND ${TOPSCC} ${file} ${compile_options} -o ${out} -ltops
+      DEPENDS ${file}
+      COMMENT "Compiling ${file} to ${out_file}"
+      VERBATIM)
+    list(APPEND outs ${out})
+  endforeach()
+  message(STATUS "== Finish compiling topscc kernels ==")
+  set(${out_libs}
+      ${outs}
+      PARENT_SCOPE)
+endfunction()

backends/gcu/common/gcu_env_list.h

@@ -21,4 +21,5 @@ namespace env {
 const char *const kUseJitKernels = "PADDLE_GCU_USE_JIT_KERNELS_ONLY";
 const char *const kProfiler = "PADDLE_GCU_PROFILE";
 const char *const kStreamAsync = "PADDLE_RUN_ASYNC";
+const char *const kEnableTransOpt = "PADDLE_ENABLE_TRANSPOSE_OPT";
 }  // namespace env

backends/gcu/common/utils.cc

@@ -142,4 +142,36 @@ bool LaunchAOTKernel(const phi::DataType& dtype,
   }
   return (supported_types.count(dtype) > 0);
 }
+
+#define FOR_EACH_DATA_TYPE_TO_PRINT(_)      \
+  _(bool, phi::DataType::BOOL)              \
+  _(int8_t, phi::DataType::INT8)            \
+  _(uint8_t, phi::DataType::UINT8)          \
+  _(int16_t, phi::DataType::INT16)          \
+  _(uint16_t, phi::DataType::UINT16)        \
+  _(int32_t, phi::DataType::INT32)          \
+  _(uint32_t, phi::DataType::UINT32)        \
+  _(int64_t, phi::DataType::INT64)          \
+  _(uint64_t, phi::DataType::UINT64)        \
+  _(phi::bfloat16, phi::DataType::BFLOAT16) \
+  _(phi::float16, phi::DataType::FLOAT16)   \
+  _(float, phi::DataType::FLOAT32)          \
+  _(double, phi::DataType::FLOAT64)
+
+#define CALL_PRINT_TENSOR(cpp_type, data_type) \
+  case data_type:                              \
+    PrintTensor<cpp_type>(os, t);              \
+    break;
+
+std::ostream& operator<<(std::ostream& os, const phi::DenseTensor& t) {
+  switch (t.dtype()) {
+    FOR_EACH_DATA_TYPE_TO_PRINT(CALL_PRINT_TENSOR)
+    default:
+      VLOG(1) << "PrintTensor unrecognized data type:" << t.dtype();
+  }
+  return os;
+}
+#undef FOR_EACH_DATA_TYPE_TO_PRINT
+#undef CALL_PRINT_TENSOR
+
 }  // namespace custom_kernel