Skip to content

Commit 3ef4ec0

Browse files
cwsmithcwsmith
authored
Merge pull request #6 from SCOREC/parallel-for-abstraction
Parallel for abstraction and initial MeshField class
2 parents 5849f46 + c9bccae commit 3ef4ec0

File tree

5 files changed

+399
-90
lines changed

5 files changed

+399
-90
lines changed

‎CMakeLists.txt‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,17 @@ include(CTest)
1616
option(IS_TESTING "Build for CTest" OFF)
1717
message(STATUS "IS_TESTING: ${IS_TESTING}")
1818

19-
add_executable(SliceWrapper test/SliceWrapper.cpp)
19+
add_executable(SliceWrapper test/testSliceWrapper.cpp)
2020
target_include_directories(SliceWrapper PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) #hack - this should be a header-only library
2121
target_link_libraries(SliceWrapper Omega_h::omega_h)
2222
target_link_libraries(SliceWrapper Cabana::cabanacore)
2323
target_compile_definitions(SliceWrapper PUBLIC ENABLE_CABANA)
2424

25-
add_test(sliceWrapper10 ./SliceWrapper 10)
26-
add_test(sliceWrapper32 ./SliceWrapper 32)
27-
add_test(sliceWrapper33 ./SliceWrapper 33)
25+
add_executable(MeshField test/testMeshField.cpp)
26+
target_include_directories(MeshField PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) #hack - this should be a header-only library
27+
target_link_libraries(MeshField Omega_h::omega_h)
28+
target_link_libraries(MeshField Cabana::cabanacore)
29+
target_compile_definitions(MeshField PUBLIC ENABLE_CABANA)
30+
2831
add_test(sliceWrapper50 ./SliceWrapper 50)
32+
add_test(meshField ./MeshField)

‎src/MeshField.hpp‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#ifndef meshfield_hpp
2+
#define meshfield_hpp
3+
4+
#include "SliceWrapper.hpp"
5+
6+
namespace MeshField {
7+
8+
template <class Slice>
9+
class Field {
10+
11+
Slice slice;
12+
13+
public:
14+
Field(Slice s) : slice(s) {}
15+
16+
KOKKOS_INLINE_FUNCTION
17+
auto& operator()(int s, int a) const {
18+
return slice.access(s,a);
19+
}
20+
21+
KOKKOS_INLINE_FUNCTION
22+
auto& operator()(int s, int a, int i) const {
23+
return slice.access(s,a,i);
24+
}
25+
26+
KOKKOS_INLINE_FUNCTION
27+
auto& operator()(int s, int a, int i, int j) const {
28+
return slice.access(s,a,i,j);
29+
}
30+
31+
KOKKOS_INLINE_FUNCTION
32+
auto& operator()(int s, int a, int i, int j, int k) const {
33+
return slice.access(s,a,i,j,k);
34+
}
35+
};
36+
37+
template <class Controller>
38+
class MeshField {
39+
40+
Controller sliceController;
41+
42+
public:
43+
MeshField(Controller sc) : sliceController(std::move(sc)) {}
44+
45+
template <std::size_t index>
46+
auto makeField() {
47+
auto slice = sliceController.template makeSlice<index>();
48+
return Field(std::move(slice));
49+
}
50+
51+
template<typename FunctorType>
52+
void parallel_for(int lower_bound, int upper_bound,
53+
FunctorType vector_kernel,
54+
std::string tag) {
55+
sliceController.parallel_for(lower_bound, upper_bound, vector_kernel, tag);
56+
}
57+
58+
};
59+
60+
}
61+
#endif

‎src/SliceWrapper.hpp‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ struct SliceWrapper {
99
SliceType st_; //store the underlying instance
1010

1111
SliceWrapper(SliceType st) : st_(st) {}
12+
13+
SliceWrapper() {}
1214

1315
KOKKOS_INLINE_FUNCTION
1416
T& access(int s, int a) const {
@@ -25,14 +27,13 @@ struct SliceWrapper {
2527
KOKKOS_INLINE_FUNCTION
2628
auto& access(int s, int a, int i, int j, int k) const {
2729
return st_.access(s,a,i,j,k);
28-
}
29-
30+
}
3031
};
3132

3233
using namespace Cabana;
3334

3435
template <class ExecutionSpace, class MemorySpace, class... Ts>
35-
class CabSliceFactory {
36+
class CabSliceController {
3637
using TypeTuple = std::tuple<Ts...>;
3738
using DeviceType = Kokkos::Device<ExecutionSpace, MemorySpace>;
3839
using DataTypes = Cabana::MemberTypes<Ts...>;
@@ -60,15 +61,23 @@ class CabSliceFactory {
6061
Cabana::AoSoA<DataTypes, DeviceType, vecLen> aosoa;
6162

6263
public:
64+
template<typename FunctorType>
65+
void parallel_for(int lower_bound, int upper_bound, FunctorType vectorKernel, std::string tag) {
66+
Cabana::SimdPolicy<vecLen, ExecutionSpace> simd_policy(lower_bound, upper_bound);
67+
Cabana::simd_parallel_for(simd_policy, vectorKernel, tag);
68+
}
69+
6370
template <std::size_t index>
64-
auto makeSliceCab() {
71+
auto makeSlice() {
6572
using type = std::tuple_element_t<index, TypeTuple>;
6673
const int stride = sizeof(soa_t) / sizeof(member_value_t<index>);
6774
auto slice = Cabana::slice<index>(aosoa);
6875
return wrapper_slice_t< type, stride >(std::move(slice));
6976
}
77+
78+
CabSliceController() {}
7079

71-
CabSliceFactory(int n) : aosoa("sliceAoSoA", n) {
80+
CabSliceController(int n) : aosoa("sliceAoSoA", n) {
7281
if (sizeof...(Ts) == 0) {
7382
throw std::invalid_argument("Must provide at least one member type in template definition");
7483
}

‎test/testMeshField.cpp‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
#include "MeshField.hpp"
2+
#include "SliceWrapper.hpp"
3+
4+
#include <Cabana_Core.hpp>
5+
6+
using ExecutionSpace = Kokkos::DefaultExecutionSpace;
7+
using MemorySpace = ExecutionSpace::memory_space;
8+
9+
10+
void single_type(int num_tuples) {
11+
using Controller = CabSliceController<ExecutionSpace, MemorySpace, double>;
12+
13+
// Slice Wrapper Controller
14+
Controller c(num_tuples);
15+
MeshField::MeshField<Controller> cabMeshField(c);
16+
17+
auto field0 = cabMeshField.makeField<0>();
18+
19+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
20+
{
21+
double d0 = 10;
22+
field0(s,a) = d0;
23+
assert(field0(s,a) == d0);
24+
};
25+
26+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"single_type_pfor");
27+
28+
}
29+
30+
void multi_type(int num_tuples) {
31+
using Controller = CabSliceController<ExecutionSpace, MemorySpace, double,
32+
double, float, int, char>;
33+
34+
// Slice Wrapper Controller
35+
Controller c(num_tuples);
36+
MeshField::MeshField<Controller> cabMeshField(c);
37+
38+
auto field0 = cabMeshField.makeField<0>();
39+
auto field1 = cabMeshField.makeField<1>();
40+
auto field2 = cabMeshField.makeField<2>();
41+
auto field3 = cabMeshField.makeField<3>();
42+
auto field4 = cabMeshField.makeField<4>();
43+
44+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
45+
{
46+
double d0 = 10.456;
47+
field0(s,a) = d0;
48+
double d1 = 43.973234567;
49+
field1(s,a) = d1;
50+
float f0 = 123.45;
51+
field2(s,a) = f0;
52+
int i0 = 22;
53+
field3(s,a) = i0;
54+
char c0 = 'a';
55+
field4(s,a) = c0;
56+
57+
assert(field0(s,a) == d0);
58+
assert(field1(s,a) == d1);
59+
assert(field2(s,a) == f0);
60+
assert(field3(s,a) == i0);
61+
assert(field4(s,a) == c0);
62+
};
63+
64+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"multi_type_pfor");
65+
}
66+
67+
void many_type(int num_tuples) {
68+
69+
using Controller = CabSliceController<ExecutionSpace, MemorySpace,
70+
double, double, float, float, int,
71+
short int, char>;
72+
73+
// Slice Wrapper Controller
74+
Controller c(num_tuples);
75+
MeshField::MeshField<Controller> cabMeshField(c);
76+
77+
auto field0 = cabMeshField.makeField<0>();
78+
auto field1 = cabMeshField.makeField<1>();
79+
auto field2 = cabMeshField.makeField<2>();
80+
auto field3 = cabMeshField.makeField<3>();
81+
auto field4 = cabMeshField.makeField<4>();
82+
auto field5 = cabMeshField.makeField<5>();
83+
auto field6 = cabMeshField.makeField<6>();
84+
85+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
86+
{
87+
double d0 = 10.456;
88+
field0(s,a) = d0;
89+
double d1 = 43.973234567;
90+
field1(s,a) = d1;
91+
float f0 = 123.45;
92+
field2(s,a) = f0;
93+
float f1 = 543.21;
94+
field3(s,a) = f1;
95+
int i0 = 222;
96+
field4(s,a) = i0;
97+
short int i1 = 50;
98+
field5(s,a) = i1;
99+
char c0 = 'h';
100+
field6(s,a) = c0;
101+
102+
assert(field0(s,a) == d0);
103+
assert(field1(s,a) == d1);
104+
assert(field2(s,a) == f0);
105+
assert(field3(s,a) == f1);
106+
assert(field4(s,a) == i0);
107+
assert(field5(s,a) == i1);
108+
assert(field6(s,a) == c0);
109+
};
110+
111+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"many_type_pfor");
112+
}
113+
114+
void rank1_arr(int num_tuples) {
115+
const int width = 3;
116+
using Controller = CabSliceController<ExecutionSpace, MemorySpace, double[width]>;
117+
118+
// Slice Wrapper Controller
119+
Controller c(num_tuples);
120+
MeshField::MeshField<Controller> cabMeshField(c);
121+
122+
auto field0 = cabMeshField.makeField<0>();
123+
124+
125+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
126+
{
127+
for (int i = 0; i < width; i++)
128+
{
129+
double d0 = 10+i;
130+
field0(s,a,i) = d0;
131+
assert(field0(s,a,i) == d0);
132+
}
133+
};
134+
135+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"rank1_arr_pfor");
136+
}
137+
138+
void rank2_arr(int num_tuples) {
139+
const int width = 3;
140+
const int height = 4;
141+
using Controller = CabSliceController<ExecutionSpace, MemorySpace,
142+
double[width][height]>;
143+
144+
// Slice Wrapper Controller
145+
Controller c(num_tuples);
146+
MeshField::MeshField<Controller> cabMeshField(c);
147+
148+
auto field0 = cabMeshField.makeField<0>();
149+
150+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
151+
{
152+
for (int i = 0; i < width; i++)
153+
{
154+
for (int j = 0; j < height; j++)
155+
{
156+
double d0 = (10+i)/(j+1);
157+
field0(s,a,i,j) = d0;
158+
assert(field0(s,a,i,j) == d0);
159+
}
160+
}
161+
};
162+
163+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"rank2_arr_pfor");
164+
}
165+
166+
void rank3_arr(int num_tuples) {
167+
const int width = 3;
168+
const int height = 4;
169+
const int depth = 2;
170+
using Controller = CabSliceController<ExecutionSpace, MemorySpace,
171+
double[width][height][depth]>;
172+
173+
// Slice Wrapper Controller
174+
Controller c(num_tuples);
175+
MeshField::MeshField<Controller> cabMeshField(c);
176+
177+
auto field0 = cabMeshField.makeField<0>();
178+
179+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
180+
{
181+
for (int i = 0; i < width; i++)
182+
{
183+
for (int j = 0; j < height; j++)
184+
{
185+
for (int k = 0; k < depth; k++)
186+
{
187+
double d0 = ((10+i)*(k+1))/(j+1);
188+
field0(s,a,i,j,k) = d0;
189+
assert(field0(s,a,i,j,k) == d0);
190+
}
191+
}
192+
}
193+
};
194+
195+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"rank3_arr_pfor");
196+
197+
}
198+
199+
void mix_arr(int num_tuples) {
200+
const int width = 3;
201+
const int height = 4;
202+
const int depth = 2;
203+
using Controller = CabSliceController<ExecutionSpace, MemorySpace,
204+
double[width][height][depth],
205+
float[width][height], int[width],
206+
char>;
207+
208+
// Slice Wrapper Controller
209+
Controller c(num_tuples);
210+
MeshField::MeshField<Controller> cabMeshField(c);
211+
212+
auto field0 = cabMeshField.makeField<0>();
213+
auto field1 = cabMeshField.makeField<1>();
214+
auto field2 = cabMeshField.makeField<2>();
215+
auto field3 = cabMeshField.makeField<3>();
216+
217+
auto vector_kernel = KOKKOS_LAMBDA(const int s, const int a)
218+
{
219+
float f0;
220+
int i0;
221+
char c0 = 's';
222+
field3(s,a) = c0;
223+
224+
for (int i = 0; i < width; i++)
225+
{
226+
i0 = i+s+a;
227+
field2(s,a,i) = i0;
228+
for (int j = 0; j < height; j++)
229+
{
230+
f0 = i0 / (i+j+1.123);
231+
field1(s,a,i,j) = f0;
232+
for (int k = 0; k < depth; k++)
233+
{
234+
double d0 = ((10+i)*(k+1))/(j+1);
235+
field0(s,a,i,j,k) = d0;
236+
assert(field0(s,a,i,j,k) == d0);
237+
}
238+
assert(field1(s,a,i,j) == f0);
239+
}
240+
assert(field2(s,a,i) == i0);
241+
}
242+
assert(field3(s,a) == c0);
243+
};
244+
245+
cabMeshField.parallel_for(0,num_tuples,vector_kernel,"mix_arr_pfor");
246+
247+
}
248+
249+
int main(int argc, char* argv[]) {
250+
int num_tuples = (argc < 2) ? (1000) : (atoi(argv[1]));
251+
Kokkos::ScopeGuard scope_guard(argc, argv);
252+
253+
single_type(num_tuples);
254+
multi_type(num_tuples);
255+
many_type(num_tuples);
256+
rank1_arr(num_tuples);
257+
rank2_arr(num_tuples);
258+
rank3_arr(num_tuples);
259+
mix_arr(num_tuples);
260+
261+
return 0;
262+
}

0 commit comments

Comments
 (0)