Skip to content

Commit 1092f09

Browse files
eebassoeebasso
committed
Merge branch 'development' of https://github.com/AMReX-Codes/amrex into test-set_eb_data
2 parents 335b267 + 9d3f012 commit 1092f09

10 files changed

Lines changed: 367 additions & 95 deletions

File tree

‎.github/workflows/cuda.yml‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,27 +69,26 @@ jobs:
6969
ccache -s
7070
du -hs ~/.cache/ccache
7171
72-
# Build libamrex and all tests for CUDA with LLVM Clang + libc++ + CTK 11.7
73-
tests-cuda11-clang:
74-
name: Clang@15 CUDA@11.7 Release [tests]
75-
runs-on: ubuntu-22.04
72+
# Build libamrex and all tests for CUDA with LLVM Clang + CTK 12.5
73+
tests-cuda-clang:
74+
name: Clang@19 CUDA@12.5 Release [tests]
75+
runs-on: ubuntu-24.04
7676
needs: check_changes
7777
if: needs.check_changes.outputs.has_non_docs_changes == 'true'
7878
env:
79-
CC: clang-15
80-
CXX: clang++-15
81-
CUDACXX: clang++-15
82-
CUDAHOSTCXX: clang++-15
83-
CUDAFLAGS: "-stdlib=libc++"
84-
LDFLAGS: "-stdlib=libc++"
85-
CUDAARCHS: "70"
86-
AMReX_CUDA_ARCH: "7.0"
79+
CC: clang-19
80+
CXX: clang++-19
81+
CUDACXX: clang++-19
82+
CUDAHOSTCXX: clang++-19
83+
CUDAARCHS: "80"
84+
AMReX_CUDA_ARCH: "8.0"
8785
steps:
8886
- uses: actions/checkout@v6
8987
- name: Dependencies
9088
run: |
9189
.github/workflows/dependencies/ubuntu_free_disk_space.sh
92-
.github/workflows/dependencies/dependencies_llvm_cuda11_clang15.sh
90+
.github/workflows/dependencies/dependencies_clang.sh 19
91+
.github/workflows/dependencies/dependencies_nvcc_2404.sh 12.5
9392
.github/workflows/dependencies/dependencies_ccache.sh
9493
- name: Set Up Cache
9594
uses: actions/cache@v5

‎.github/workflows/dependencies/dependencies_llvm_cuda11_clang15.sh‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎Src/Base/AMReX_Print.H‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ namespace amrex
7373
~Print () {
7474
if (to_print) {
7575
os.flush();
76-
os << ss.str();
76+
os << ss.view();
7777
os.flush();
7878
}
7979
}
@@ -148,7 +148,7 @@ namespace amrex
148148
~PrintToFile () {
149149
if (to_print) {
150150
ofs.flush();
151-
ofs << ss.str();
151+
ofs << ss.view();
152152
ofs.flush();
153153
}
154154
}

‎Src/Particle/AMReX_NeighborParticlesGPUImpl.H‎

Lines changed: 93 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,65 @@ namespace amrex {
77
/// \cond DOXYGEN_IGNORE
88
namespace detail
99
{
10+
template <typename F>
11+
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
12+
void forEachIntersectingTile (IntVect const& iv, int nGrow,
13+
Box const& grid_box, IntVect const& periodic_shift,
14+
bool do_tiling, IntVect const& tile_size, F&& f)
15+
{
16+
Box cell_box(iv, iv);
17+
cell_box.grow(nGrow);
18+
cell_box += periodic_shift;
19+
cell_box &= grid_box;
20+
21+
if (!cell_box.ok()) { return; }
22+
23+
auto const& cb_lo = cell_box.smallEnd();
24+
auto const& cb_hi = cell_box.bigEnd();
25+
26+
#if (AMREX_SPACEDIM == 1)
27+
for (int i = cb_lo[0]; i <= cb_hi[0]; ++i) {
28+
IntVect cell(AMREX_D_DECL(i, 0, 0));
29+
Box tbx;
30+
int tile = getTileIndex(cell, grid_box, do_tiling, tile_size, tbx);
31+
IntVect rep(AMREX_D_DECL(amrex::max(cb_lo[0], tbx.smallEnd(0)), 0, 0));
32+
if (cell == rep) {
33+
f(tile);
34+
}
35+
}
36+
#elif (AMREX_SPACEDIM == 2)
37+
for (int j = cb_lo[1]; j <= cb_hi[1]; ++j) {
38+
for (int i = cb_lo[0]; i <= cb_hi[0]; ++i) {
39+
IntVect cell(AMREX_D_DECL(i, j, 0));
40+
Box tbx;
41+
int tile = getTileIndex(cell, grid_box, do_tiling, tile_size, tbx);
42+
IntVect rep(AMREX_D_DECL(amrex::max(cb_lo[0], tbx.smallEnd(0)),
43+
amrex::max(cb_lo[1], tbx.smallEnd(1)),
44+
0));
45+
if (cell == rep) {
46+
f(tile);
47+
}
48+
}
49+
}
50+
#else
51+
for (int k = cb_lo[2]; k <= cb_hi[2]; ++k) {
52+
for (int j = cb_lo[1]; j <= cb_hi[1]; ++j) {
53+
for (int i = cb_lo[0]; i <= cb_hi[0]; ++i) {
54+
IntVect cell(AMREX_D_DECL(i, j, k));
55+
Box tbx;
56+
int tile = getTileIndex(cell, grid_box, do_tiling, tile_size, tbx);
57+
IntVect rep(AMREX_D_DECL(amrex::max(cb_lo[0], tbx.smallEnd(0)),
58+
amrex::max(cb_lo[1], tbx.smallEnd(1)),
59+
amrex::max(cb_lo[2], tbx.smallEnd(2))));
60+
if (cell == rep) {
61+
f(tile);
62+
}
63+
}
64+
}
65+
}
66+
#endif
67+
}
68+
1069
inline Vector<Box> getBoundaryBoxes(const Box& box, int ncells)
1170
{
1271
AMREX_ASSERT_WITH_MESSAGE(box.size() > 2*IntVect(AMREX_D_DECL(ncells, ncells, ncells)),
@@ -86,7 +145,6 @@ buildNeighborMask ()
86145
{
87146
int nbor_grid = isec.first;
88147
const Box isec_box = isec.second - pshift;
89-
if ( (grid == nbor_grid) && (pshift == 0)) { continue; }
90148
neighbor_grids.insert(NeighborTask(nbor_grid, isec_box, pshift));
91149
const int global_rank = dmap[nbor_grid];
92150
neighbor_procs.push_back(ParallelContext::global_to_local_rank(global_rank));
@@ -173,7 +231,7 @@ buildNeighborCopyOp (bool use_boundary_neighbor)
173231
const int nisec_box = m_isec_boxes[gid].size();
174232
const bool do_tiling = this->do_tiling;
175233
const IntVect tile_size = this->tile_size;
176-
// auto p_code_offsets = m_code_offsets[gid].dataPtr();
234+
const int nGrow = m_num_neighbor_cells;
177235

178236
AMREX_FOR_1D ( np, i,
179237
{
@@ -186,7 +244,20 @@ buildNeighborCopyOp (bool use_boundary_neighbor)
186244
IntVect iv = getParticleCell(p_ptr[pid], plo, dxi, domain);
187245
for (int j=0; j<nisec_box; ++j) {
188246
if (p_isec_boxes[j].contains(iv)) {
189-
++p_counts[i];
247+
detail::forEachIntersectingTile(iv, nGrow,
248+
p_code_array[j].grid_box,
249+
p_code_array[j].periodic_shift,
250+
do_tiling, tile_size,
251+
[&] (int dst_tile)
252+
{
253+
bool is_self = (p_code_array[j].grid_id == gid) && (dst_tile == tid)
254+
AMREX_D_TERM( && (p_code_array[j].periodic_shift[0] == 0),
255+
&& (p_code_array[j].periodic_shift[1] == 0),
256+
&& (p_code_array[j].periodic_shift[2] == 0));
257+
if (!is_self) {
258+
++p_counts[i];
259+
}
260+
});
190261
}
191262
}
192263
});
@@ -217,14 +288,25 @@ buildNeighborCopyOp (bool use_boundary_neighbor)
217288
int k = p_offsets[i];
218289
for (int j=0; j<nisec_box; ++j) {
219290
if (p_isec_boxes[j].contains(iv)) {
220-
p_boxes[k] = p_code_array[j].grid_id;
221-
Box tbx;
222-
p_tiles[k] = getTileIndex(iv, p_code_array[j].grid_box,
223-
do_tiling, tile_size, tbx);
224-
p_levs[k] = 0;
225-
p_periodic_shift[k] = p_code_array[j].periodic_shift;
226-
p_src_indices[k] = pid;
227-
++k;
291+
detail::forEachIntersectingTile(iv, nGrow,
292+
p_code_array[j].grid_box,
293+
p_code_array[j].periodic_shift,
294+
do_tiling, tile_size,
295+
[&] (int dst_tile)
296+
{
297+
bool is_self = (p_code_array[j].grid_id == gid) && (dst_tile == tid)
298+
AMREX_D_TERM( && (p_code_array[j].periodic_shift[0] == 0),
299+
&& (p_code_array[j].periodic_shift[1] == 0),
300+
&& (p_code_array[j].periodic_shift[2] == 0));
301+
if (!is_self) {
302+
p_boxes[k] = p_code_array[j].grid_id;
303+
p_tiles[k] = dst_tile;
304+
p_levs[k] = 0;
305+
p_periodic_shift[k] = p_code_array[j].periodic_shift;
306+
p_src_indices[k] = pid;
307+
++k;
308+
}
309+
});
228310
}
229311
}
230312
AMREX_ALWAYS_ASSERT(k == p_offsets[i+1]);
@@ -243,9 +325,6 @@ fillNeighborsGPU ()
243325

244326
AMREX_ASSERT(numParticlesOutOfRange(*this, 0) == 0);
245327

246-
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(this->do_tiling == 0,
247-
"Tiling on the GPU is not supported for neighbor particles.");
248-
249328
buildNeighborMask();
250329
this->defineBufferMap();
251330

‎Src/Particle/AMReX_NeighborParticlesI.H‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,8 +1005,8 @@ selectActualNeighbors (CheckPair const& check_pair, int num_cells)
10051005
const auto* pstruct = aos().dataPtr();
10061006
const auto ptile_data = this->ParticlesAt(lev, pti).getConstParticleTileData();
10071007

1008-
Box box = pti.validbox();
1009-
Box grownBox = pti.tilebox();
1008+
Box box = pti.tilebox();
1009+
Box grownBox = box;
10101010
grownBox.grow(computeRefFac(0, lev).max()*m_num_neighbor_cells);
10111011
const auto lo = lbound(grownBox);
10121012
const auto hi = ubound(grownBox);

‎Src/Particle/AMReX_ParticleUtil.H‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ numParticlesOutOfRange (Iterator const& pti, IntVect nGrow)
9595
*
9696
*/
9797
template <class PC, std::enable_if_t<IsParticleContainer<PC>::value, int> foo = 0>
98-
int
98+
Long
9999
numParticlesOutOfRange (PC const& pc, int nGrow)
100100
{
101101
return numParticlesOutOfRange(pc, 0, pc.finestLevel(), nGrow);
@@ -114,7 +114,7 @@ numParticlesOutOfRange (PC const& pc, int nGrow)
114114
*
115115
*/
116116
template <class PC, std::enable_if_t<IsParticleContainer<PC>::value, int> foo = 0>
117-
int
117+
Long
118118
numParticlesOutOfRange (PC const& pc, IntVect nGrow)
119119
{
120120
return numParticlesOutOfRange(pc, 0, pc.finestLevel(), nGrow);
@@ -135,7 +135,7 @@ numParticlesOutOfRange (PC const& pc, IntVect nGrow)
135135
*
136136
*/
137137
template <class PC, std::enable_if_t<IsParticleContainer<PC>::value, int> foo = 0>
138-
int
138+
Long
139139
numParticlesOutOfRange (PC const& pc, int lev_min, int lev_max, int nGrow)
140140
{
141141
BL_PROFILE("numParticlesOutOfRange()");
@@ -159,13 +159,13 @@ numParticlesOutOfRange (PC const& pc, int lev_min, int lev_max, int nGrow)
159159
*
160160
*/
161161
template <class PC, std::enable_if_t<IsParticleContainer<PC>::value, int> foo = 0>
162-
int
162+
Long
163163
numParticlesOutOfRange (PC const& pc, int lev_min, int lev_max, IntVect nGrow)
164164
{
165165
BL_PROFILE("numParticlesOutOfRange()");
166166

167167
using ParIter = typename PC::ParConstIterType;
168-
int num_wrong = 0;
168+
Long num_wrong = 0;
169169
for (int lev = lev_min; lev <= lev_max; ++lev)
170170
{
171171
#ifdef AMREX_USE_OMP

‎Tests/Particles/NeighborParticles/MDParticleContainer.H‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public:
4545

4646
void reset_test_id ();
4747

48-
void checkNeighborParticles ();
48+
void checkNeighborParticles (bool use_source_grid = true);
4949

5050
void checkNeighborList ();
5151

0 commit comments

Comments
 (0)