Skip to content

Commit 1cec840

Browse files
preshanthpreshanth
committed
Merge branch 'kinit2' into 'main'
KokkosInit.h: Code for selective initialize/finalize Kokkos backends. See merge request ardg/libra!148
2 parents 1bd7869 + 970c7f3 commit 1cec840

22 files changed

+682
-117
lines changed

‎apps/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,10 @@ set (RESTORE_MDSPAN_SOURCES
181181
set (ROADRUNNER_SOURCES
182182
RoadRunner/roadrunner.cc
183183
RoadRunner/roadrunner.h
184+
RoadRunner/LibHPG.h
185+
RoadRunner/LibHPG.cc
186+
RoadRunner/KokkosInit.cc
187+
RoadRunner/KokkosInit.h
184188
# RoadRunner/DataBase.h
185189
# RoadRunner/DataIterations.h
186190
# RoadRunner/MakeComponents.h

‎apps/src/Acme/acme_cl_interface.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ void UI(Bool restart, int argc, char **argv, bool interactive,
7777
//
7878
//-------------------------------------------------------------------------
7979
//
80+
#ifndef ACME_LIBRARY_BUILD
8081
int main(int argc, char **argv)
8182
{
8283
//
@@ -118,3 +119,4 @@ int main(int argc, char **argv)
118119
}
119120
// if (restartUI) RestartUI(RENTER);
120121
}
122+
#endif // ACME_LIBRARY_BUILD

‎apps/src/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ foreach (APP_DIR IN LISTS APP_LIST)
3535
# Build individual CLI interface library for unit tests if the source file exists
3636
if(Apps_BUILD_TESTS AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${APP_DIR}/${APP_NAME}_cl_interface.cc")
3737
add_library(${APP_NAME}_cl_interface SHARED "${CMAKE_CURRENT_SOURCE_DIR}/${APP_DIR}/${APP_NAME}_cl_interface.cc")
38+
string(TOUPPER ${APP_NAME} APP_NAME_UPPER)
39+
target_compile_definitions(${APP_NAME}_cl_interface PRIVATE ${APP_NAME_UPPER}_LIBRARY_BUILD)
3840
target_link_libraries(${APP_NAME}_cl_interface PRIVATE ${APP_LINK_LIBRARIES} libra "-Wl,--disable-new-dtags")
3941
install(TARGETS ${APP_NAME}_cl_interface
4042
LIBRARY DESTINATION "${CMAKE_INSTALL_PREFIX}/lib")

‎apps/src/Chip/chip_cl_interface.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ void UI(Bool restart, int argc, char **argv, bool interactive,
8383
//
8484
//-------------------------------------------------------------------------
8585
//
86+
#ifndef CHIP_LIBRARY_BUILD
8687
int main(int argc, char **argv)
8788
{
8889
//
@@ -129,3 +130,4 @@ int main(int argc, char **argv)
129130
}
130131
// if (restartUI) RestartUI(RENTER);
131132
}
133+
#endif // CHIP_LIBRARY_BUILD

‎apps/src/Coyote/coyote_cl_interface.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ void UI(bool restart, int argc, char **argv, bool interactive,
7171
}
7272
// else
7373
// clRetry();
74-
REENTER:
7574
try
7675
{
7776
SMap watchPoints; VString exposedKeys;
@@ -151,6 +150,7 @@ void UI(bool restart, int argc, char **argv, bool interactive,
151150
//
152151
//--------------------------------------------------------------------------
153152
//
153+
#ifndef COYOTE_LIBRARY_BUILD
154154
int main(int argc, char **argv)
155155
{
156156
//
@@ -215,6 +215,7 @@ int main(int argc, char **argv)
215215
}
216216
return 0;
217217
}
218+
#endif // COYOTE_LIBRARY_BUILD
218219
//
219220
//--------------------------------------------------------------------------
220221
//

‎apps/src/Dale/dale_cl_interface.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ void UI(bool restart, int argc, char **argv, bool interactive,
9797
//
9898
//-------------------------------------------------------------------------
9999
//
100+
#ifndef DALE_LIBRARY_BUILD
100101
int main(int argc, char **argv)
101102
{
102103
//
@@ -152,3 +153,4 @@ int main(int argc, char **argv)
152153
}
153154
if (restartUI) RestartUI(RENTER);
154155
}
156+
#endif // DALE_LIBRARY_BUILD

‎apps/src/Hummbee/hummbee_cl_interface.cc‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,6 @@ void UI(bool restart, int argc, char **argv, bool interactive,
8888
}
8989
// else
9090
// clRetry();
91-
REENTER:
9291
try
9392
{
9493
SMap watchPoints; VString exposedKeys;
@@ -180,7 +179,7 @@ void UI(bool restart, int argc, char **argv, bool interactive,
180179
}
181180
}
182181

183-
182+
#ifndef HUMMBEE_LIBRARY_BUILD
184183
int main(int argc, char **argv)
185184
{
186185
//
@@ -264,3 +263,4 @@ int main(int argc, char **argv)
264263

265264
return 0;
266265
}
266+
#endif // HUMMBEE_LIBRARY_BUILD

‎apps/src/MSSplit/mssplit_cl_interface.cc‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ void UI(Bool restart, int argc, char **argv, bool interactive, string& MSNBuf, s
3333
}
3434
//else
3535
// clRetry();
36-
REENTER:
36+
//REENTER:
3737
try
3838
{
3939
int i;
@@ -71,6 +71,7 @@ void UI(Bool restart, int argc, char **argv, bool interactive, string& MSNBuf, s
7171
}
7272
}
7373

74+
#ifndef MSSPLIT_LIBRARY_BUILD
7475
int main(int argc, char **argv)
7576
{
7677
//
@@ -83,7 +84,7 @@ int main(int argc, char **argv)
8384
Bool deepCopy=0;
8485
Bool restartUI=False;
8586

86-
REENTER:// UI re-entry point.
87+
//REENTER:// UI re-entry point.
8788
//
8889
//---------------------------------------------------
8990
//
@@ -134,7 +135,8 @@ int main(int argc, char **argv)
134135
}
135136

136137
showTableCache();
137-
//if (restartUI)
138+
//if (restartUI)
138139
//restartUI=True;
139140
//RestartUI(REENTER);
140141
}
142+
#endif // MSSPLIT_LIBRARY_BUILD

‎apps/src/RoadRunner/KokkosInit.cc‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/* Copyright 2024 Stanford University, NVIDIA Corporation
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
#include <KokkosInit.h>
16+
namespace KokkosInterop
17+
{
18+
std::string kokkos_backend_to_str(const KokkosInterop::K_BACKEND& dev)
19+
{
20+
std::string str;
21+
if (dev == KokkosInterop::K_BACKEND::K_CUDA) str="K::Cuda";
22+
else if (dev == KokkosInterop::K_BACKEND::K_SERIAL) str="K::Serial";
23+
else if (dev == KokkosInterop::K_BACKEND::K_OPENMP) str="K::OpenMP";
24+
else if (dev == KokkosInterop::K_BACKEND::K_HIP) str="K::HIP";
25+
26+
return str;
27+
}
28+
29+
//
30+
//------------------------------------------------------------------
31+
// Input: A list of Kokkos backends to initialize.
32+
// Output: A list of Kokkos backends that were successfully initialized.
33+
//
34+
std::vector<KokkosInterop::K_BACKEND>
35+
kokkos_initialize(const std::vector<KokkosInterop::K_BACKEND>& K_Device)
36+
{
37+
std::vector<KokkosInterop::K_BACKEND> list_of_backends;
38+
39+
// use Kokkos::Impl::{pre,post}_initialize to allow us to do our own
40+
// execution space initialization
41+
Kokkos::InitializationSettings kokkos_init_args;
42+
//os << "doing general pre-initialization" << endl;
43+
Kokkos::Impl::pre_initialize(kokkos_init_args);
44+
45+
for(auto BE : K_Device)
46+
switch(BE)
47+
{
48+
#ifdef KOKKOS_ENABLE_SERIAL
49+
case K_SERIAL:
50+
{
51+
Kokkos::Serial::impl_initialize(kokkos_init_args);
52+
if (Kokkos::Serial::impl_is_initialized())
53+
list_of_backends.push_back(K_SERIAL);
54+
break;
55+
};
56+
#endif
57+
#ifdef KOKKOS_ENABLE_OPENMP
58+
case K_OPENMP:
59+
{
60+
KokkosOpenMPInitializer ompinit;
61+
if (ompinit.execute_on_processor())
62+
list_of_backends.push_back(K_OPENMP);
63+
ompinit.wait_done();
64+
break;
65+
};
66+
#endif
67+
#ifdef KOKKOS_ENABLE_CUDA
68+
case K_CUDA:
69+
{
70+
KokkosCudaInitializer cudainit;
71+
if (cudainit.execute_on_processor())
72+
list_of_backends.push_back(K_CUDA);
73+
cudainit.wait_done();
74+
break;
75+
};
76+
#endif
77+
#ifdef KOKKOS_ENABLE_HIP
78+
case K_HIP:
79+
{
80+
KokkosHIPInitializer hipinit;
81+
if (hipinit.execute_on_processor())
82+
list_of_backends.push_back(K_HIP);
83+
hipinit.wait_done();
84+
break;
85+
};
86+
#endif
87+
};
88+
// TODO: warn if Kokkos has other execution spaces enabled that we're not
89+
// willing/able to initialize?
90+
91+
//os << "doing general post-initialization" << endl;
92+
Kokkos::Impl::post_initialize(kokkos_init_args);
93+
//global_initialized_kokkos_backends = list_of_backends;
94+
return list_of_backends;
95+
}
96+
//
97+
//------------------------------------------------------------------
98+
//
99+
void kokkos_finalize()
100+
{
101+
#if KOKKOS_VERSION >= 40000
102+
Kokkos::Impl::pre_finalize();
103+
#endif
104+
105+
// !!!! Does the Serial backend also need finalization?
106+
107+
#ifdef KOKKOS_ENABLE_OPENMP
108+
if (Kokkos::OpenMP::impl_is_initialized())
109+
{
110+
KokkosOpenMPFinalizer ompfinal;
111+
ompfinal.execute_on_processor();
112+
ompfinal.wait_done();
113+
}
114+
#endif
115+
116+
#ifdef KOKKOS_ENABLE_CUDA
117+
if (Kokkos::Cuda::impl_is_initialized())
118+
{
119+
KokkosCudaFinalizer cudafinal;
120+
cudafinal.execute_on_processor();
121+
cudafinal.wait_done();
122+
}
123+
#endif
124+
125+
#ifdef KOKKOS_ENABLE_HIP
126+
if (Kokkos::HIP::impl_is_initialized())
127+
{
128+
KokkosHIPFinalizer hipfinal;
129+
hipfinal.execute_on_processor();
130+
hipfinal.wait_done();
131+
}
132+
#endif
133+
134+
// cerr << "doing general finalization" << endl;
135+
// #if KOKKOS_VERSION >= 40000
136+
// Kokkos::Impl::post_finalize();
137+
// #else
138+
// Kokkos::finalize();
139+
// #endif
140+
}
141+
};
142+

0 commit comments

Comments
 (0)