Skip to content

Commit f6b03df

Browse files
author
kmcgrie
authored
Merge pull request #64 from oneapi-src/dev_svm_interop_fix
[SVM][SYCL][Nvidia] Updated cmake and interop call fix to cublas.
2 parents 721b156 + 495b9b7 commit f6b03df

File tree

2 files changed

+54
-47
lines changed

2 files changed

+54
-47
lines changed

‎svm/SYCL/CMakeLists.txt‎

Lines changed: 45 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
# MIT License
32

43
# Copyright (c) 2015 University of West Bohemia
@@ -60,7 +59,7 @@ set(CMAKE_CXX_EXTENSIONS OFF) # Use -std, not -gnu
6059

6160
option(GPU_AOT "Build AOT for Intel GPU" OFF)
6261
option(USE_NVIDIA_BACKEND "Build for NVIDIA backend" OFF)
63-
option(USE_AMDHIP_BACKEND "Build for AMD HIP backend" OFF)
62+
option(USE_AMD_BACKEND "Build for AMD HIP backend" OFF)
6463

6564
set(SOURCES
6665
cuSVM/cuSVMSolver.dp.cpp
@@ -76,28 +75,39 @@ set(SOURCES
7675
#infrastructure/SYCL.cpp
7776
)
7877

79-
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
80-
8178
include_directories(${CMAKE_SOURCE_DIR}
8279
${CMAKE_SOURCE_DIR}/libSVM
8380
${CMAKE_SOURCE_DIR}/cuSVM
8481
${CMAKE_SOURCE_DIR}/infrastructure)
8582

8683

87-
8884
# Use either default or user defined CXX flags
8985
# -DCMAKE_CXX_FLAGS=" -blah -blah " overrides the default flags
86+
set(DEF_INTEL_WL_CXX_FLAGS " -DMKL_ILP64 ")
87+
set(DEF_NVIDIA_WL_CXX_FLAGS " -DUSE_CUBLAS ")
88+
set(DEF_AMD_WL_CXX_FLAGS " -DUSE_HIPBLAS ")
9089

91-
set(INTEL_GPU_CXX_FLAGS " -O3 -fsycl")
92-
set(NVIDIA_GPU_CXX_FLAGS " -O3 -fsycl -DUSE_CUBLAS")
93-
set(AMD_GPU_CXX_FLAGS " -O3 -fsycl -DUSE_HIPBLAS -D__HIP_PLATFORM_AMD__")
90+
set(DEF_INTEL_GENERAL_CXX_FLAGS " -O3 -fsycl ")
91+
set(DEF_NVIDIA_GENERAL_CXX_FLAGS " -O3 -fsycl ")
92+
set(DEF_AMD_GENERAL_CXX_FLAGS " -O3 -fsycl -D__HIP_PLATFORM_AMD__ ")
9493

95-
set(USE_DEFAULT_FLAGS ON)
96-
if("${CMAKE_CXX_FLAGS}" STREQUAL "")
94+
if(NOT "${CMAKE_CXX_FLAGS}" STREQUAL "" AND NOT "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
95+
message(FATAL_ERROR "Both CMAKE_CXX_FLAGS and OVERRIDE_GENERAL_CXX_FLAGS cannot be passed in together")
96+
elseif("${CMAKE_CXX_FLAGS}" STREQUAL "" AND "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
9797
message(STATUS "Using DEFAULT compilation flags")
98-
else()
99-
message(STATUS "OVERRIDING DEFAULT compilation flags")
100-
set(USE_DEFAULT_FLAGS OFF)
98+
set(INTEL_GPU_CXX_FLAGS "${DEF_INTEL_GENERAL_CXX_FLAGS} ${DEF_INTEL_WL_CXX_FLAGS}")
99+
set(NVIDIA_GPU_CXX_FLAGS "${DEF_NVIDIA_GENERAL_CXX_FLAGS} ${DEF_NVIDIA_WL_CXX_FLAGS}")
100+
set(AMD_GPU_CXX_FLAGS "${DEF_AMD_GENERAL_CXX_FLAGS} ${DEF_AMD_WL_CXX_FLAGS}")
101+
elseif(NOT "${OVERRIDE_GENERAL_CXX_FLAGS}" STREQUAL "")
102+
message(STATUS "OVERRIDING GENERAL compilation flags")
103+
set(INTEL_GPU_CXX_FLAGS "${OVERRIDE_GENERAL_CXX_FLAGS} ${DEF_INTEL_WL_CXX_FLAGS}")
104+
set(NVIDIA_GPU_CXX_FLAGS "${OVERRIDE_GENERAL_CXX_FLAGS} ${DEF_NVIDIA_WL_CXX_FLAGS}")
105+
set(AMD_GPU_CXX_FLAGS "${OVERRIDE_GENERAL_CXX_FLAGS} ${DEF_AMD_WL_CXX_FLAGS}")
106+
elseif(NOT "${CMAKE_CXX_FLAGS}" STREQUAL "")
107+
message(STATUS "OVERRIDING GENERAL and WORKLOAD SPECIFIC compilation flags")
108+
set(INTEL_GPU_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
109+
set(NVIDIA_GPU_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
110+
set(AMD_GPU_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
101111
endif()
102112

103113
#add_compile_options(-DRUN_ON_GPU)
@@ -112,30 +122,32 @@ elseif(USE_AMD_BACKEND)
112122
target_link_libraries(${PROJECT_NAME} -lhipblas)
113123
else()
114124
message(STATUS "Enabling INTEL backend")
115-
find_package(oneMKL REQUIRED CONFIG HINTS ${oneMKLROOT})
116-
message(STATUS "Found oneMKL: ${oneMKL_DIR}")
117-
target_link_libraries(${PROJECT_NAME} PRIVATE MKL::onemkl)
125+
link_directories(${MKLROOT}/lib/intel64)
126+
target_link_libraries(${PROJECT_NAME} mkl_sycl mkl_intel_ilp64 mkl_tbb_thread mkl_core pthread dl m)
118127
endif()
119128

120-
if(GPU_AOT)
129+
if(GPU_AOT)
121130
message(STATUS "Enabling INTEL backend")
122-
if(USE_DEFAULT_FLAGS)
123-
set(CMAKE_CXX_FLAGS "${INTEL_GPU_CXX_FLAGS}") # Default flags for Intel backend
131+
set(CMAKE_CXX_FLAGS "${INTEL_GPU_CXX_FLAGS}")
132+
if( (${GPU_AOT} STREQUAL "pvc") OR (${GPU_AOT} STREQUAL "PVC") )
133+
message(STATUS "Enabling Intel GPU AOT compilation for ${GPU_AOT}")
134+
string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=spir64_gen -Xs \"-device 0x0bd5 -revision_id 3\" ")
135+
else()
136+
message(STATUS "Using custom AOT compilation flag ${GPU_AOT}")
137+
string(APPEND CMAKE_CXX_FLAGS " ${GPU_AOT} ") # User should be aware of advanced AOT compilation flags
124138
endif()
125-
message(STATUS "Using custom AOT compilation flag ${GPU_AOT}")
126-
string(APPEND CMAKE_CXX_FLAGS " ${GPU_AOT} ") # User should be aware of advanced AOT compilation flags
127-
elseif(USE_NVIDIA_BACKEND)
139+
elseif(USE_NVIDIA_BACKEND)
128140
message(STATUS "Enabling NVIDIA backend")
129-
if(USE_DEFAULT_FLAGS)
130-
set(CMAKE_CXX_FLAGS "${NVIDIA_GPU_CXX_FLAGS}") # Default flags for NV backend
131-
endif()
132-
string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=nvidia_gpu_sm_${USE_SM}")
133-
elseif(USE_AMD_BACKEND)
134-
message(STATUS "Enabling AMD HIP backend.")
135-
if(USE_DEFAULT_FLAGS)
136-
set(CMAKE_CXX_FLAGS "${AMD_GPU_CXX_FLAGS}") # Default flags for AMD backend (gfx908 for MI100)
137-
endif()
138-
string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=amd_gpu_gfx${USE_AMD_ARCH}")
141+
set(CMAKE_CXX_FLAGS "${NVIDIA_GPU_CXX_FLAGS}")
142+
string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=nvidia_gpu_sm_${USE_SM} ")
143+
elseif(USE_AMDHIP_BACKEND)
144+
message(STATUS "Enabling AMD HIP backend for ${USE_AMDHIP_BACKEND} AMD architecture")
145+
set(CMAKE_CXX_FLAGS "${AMD_GPU_CXX_FLAGS}")
146+
string(APPEND CMAKE_CXX_FLAGS " -fsycl-targets=amd_gpu_gfx${USE_AMD_ARCH} ")
147+
else()
148+
# JIT case
149+
message(STATUS "Enabling INTEL backend")
150+
set(CMAKE_CXX_FLAGS "${INTEL_GPU_CXX_FLAGS}")
139151
endif()
140152

141153
# Output the compiler flags that were constructed for visual inspection
@@ -144,4 +156,4 @@ file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/a9a
144156
DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
145157

146158
#add_executable(${PROJECT_NAME} ${SOURCES})
147-
target_link_libraries(${PROJECT_NAME} PRIVATE sycl stdc++fs)
159+
target_link_libraries(${PROJECT_NAME} sycl stdc++fs)

‎svm/SYCL/cuSVM/cuSVMSolver.dp.cpp‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -491,19 +491,19 @@ float *d_SelfDotProd,const int& m,const int& n,const int &nbrCtas,const int& thr
491491
q_ct1.submit([&](sycl::handler &cgh) {
492492
//auto d_A = b_A.get_access<sycl::access::mode::read_write>(cgh);
493493
cgh.host_task([=](sycl::interop_handle ih) {
494-
auto cudaStreamHandle = sycl::get_native<sycl::backend::ext_oneapi_cuda>(q_ct1);
495-
cublasSetStream(handle, cudaStreamHandle);
496-
//auto cuA = reinterpret_cast<float *>(ih.get_mem<sycl::backend::ext_oneapi_cuda>(d_A));
497-
//constexpr float ALPHA = 2.f;
498-
//constexpr int INCX = 1;
494+
cuCtxSetCurrent(ih.get_native_context<sycl::backend::ext_oneapi_cuda>());
495+
auto cuStream = ih.get_native_queue<sycl::backend::ext_oneapi_cuda>();
496+
cublasSetStream(handle, cuStream);
499497
constexpr float ALPHA = 1.0f;
500498
constexpr float BETA = 0.0f;
501499
CHECK_ERROR(cublasSgemv (handle, CUBLAS_OP_N, m, n, &ALPHA, d_x, m, d_Kernel_InterRow, 1, &BETA, d_KernelDotProd, 1));
502-
cublasDestroy(handle);
503-
cudaStreamSynchronize(cudaStreamHandle);
500+
cudaStreamSynchronize(cuStream);
504501
//cudaDeviceSynchronize();
505502
});
506-
});
503+
}).wait_and_throw();
504+
505+
cublasDestroy(handle);
506+
507507
#elif USE_HIPBLAS
508508

509509
constexpr float ALPHA = 1.0f;
@@ -1317,9 +1317,4 @@ _kernelwidth*=-1;
13171317
mxCUDA_SAFE_CALL((sycl::free(d_KernelDotProd, q_ct1), 0));
13181318

13191319
//return;
1320-
}
1321-
//catch (sycl::exception const &exc) {
1322-
// std::cerr << exc.what() << "Exception caught at file:" << __FILE__
1323-
// << ", line:" << __LINE__ << std::endl;
1324-
//std::exit(1);
1325-
//}
1320+
}

0 commit comments

Comments
 (0)