From 3f5099ac466341d44e18763d2cb1840a9046560e Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 12:59:59 +0200 Subject: [PATCH 01/32] Add CI rules for ALPS --- ci/cscs-gh200.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ci/cscs-gh200.yml diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml new file mode 100644 index 00000000..44d71c6e --- /dev/null +++ b/ci/cscs-gh200.yml @@ -0,0 +1,13 @@ +job1: + extends: .uenv-runner-daint-gh200 + image: julia/25.5:v1 + script: + - echo $CUDA_VISIBLE_DEVICES + - echo $SLURM_LOCALID + - julia --project -e 'using InteractiveUtils; versioninfo()' + - julia --project -e 'using Pkg; Pkg.status()' + variables: + WITH_UENV_VIEW: 'juliaup, modules' + SLURM_JOB_NUM_NODES: 2 + SLURM_NTASKS_PER_NODE: 4 + SLURM_GPUS_PER_TASK: 1 From 383f33bf4e5ec8614ec9ad220f6326280ddae8c3 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 13:06:56 +0200 Subject: [PATCH 02/32] Update --- ci/cscs-gh200.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 44d71c6e..908e5d9b 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -7,7 +7,7 @@ job1: - julia --project -e 'using InteractiveUtils; versioninfo()' - julia --project -e 'using Pkg; Pkg.status()' variables: - WITH_UENV_VIEW: 'juliaup, modules' + WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 SLURM_GPUS_PER_TASK: 1 From e8732401d9ff716d1629a4b16c5e862a785496c8 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 13:12:31 +0200 Subject: [PATCH 03/32] Fixup --- ci/cscs-gh200.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 908e5d9b..7d0d7c8f 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -1,6 +1,6 @@ job1: extends: .uenv-runner-daint-gh200 - image: julia/25.5:v1 + image: julia/25.5:v1 script: - echo $CUDA_VISIBLE_DEVICES - echo $SLURM_LOCALID From 632f44fa2fb19b17ebd19558bbc264a913c1befc Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 14:10:23 +0200 Subject: [PATCH 04/32] Fixup --- ci/cscs-gh200.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 7d0d7c8f..e3698a71 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -1,3 +1,6 @@ +include: + - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' + job1: extends: .uenv-runner-daint-gh200 image: julia/25.5:v1 From 4fe15a43280d4481dc6099520aa308166d558c2b Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 14:45:17 +0200 Subject: [PATCH 05/32] Add more testing --- ci/cscs-gh200.yml | 25 +++++++++++++++++++----- scripts/single_particle_advection_MPI.jl | 5 ++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index e3698a71..51c3bf36 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -1,16 +1,31 @@ include: - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' -job1: +unit_test: extends: .uenv-runner-daint-gh200 image: julia/25.5:v1 script: - - echo $CUDA_VISIBLE_DEVICES - - echo $SLURM_LOCALID - - julia --project -e 'using InteractiveUtils; versioninfo()' - - julia --project -e 'using Pkg; Pkg.status()' + - julia -e 'println("Instantiating project") + using Pkg + Pkg.develop(; path=pwd())' + - julia -e 'println("Running tests") + using Pkg + Pkg.test("JustPIC"; test_args=["--backend=CUDA"])' + variables: + WITH_UENV_VIEW: 'juliaup' + SLURM_JOB_NUM_NODES: 1 + SLURM_NTASKS_PER_NODE: 1 + SLURM_GPUS_PER_TASK: 1 + SLURM_TIMELIMIT: "00:30:00" + +ref_test: + extends: .uenv-runner-daint-gh200 + image: julia/25.5:v1 + script: + - julia --project scripts/single_particle_advection_MPI.jl variables: WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 SLURM_GPUS_PER_TASK: 1 + SLURM_TIMELIMIT: "00:30:00" diff --git a/scripts/single_particle_advection_MPI.jl b/scripts/single_particle_advection_MPI.jl index 9be94933..c9786f29 100644 --- a/scripts/single_particle_advection_MPI.jl +++ b/scripts/single_particle_advection_MPI.jl @@ -4,7 +4,10 @@ using JustPIC._2D # Threads is the default backend, # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") -const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +# const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend + +using CUDA +const backend = JustPIC.CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend using GLMakie From 124e4debb25b487a6a756ee10a56bbc4f3b89bc4 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 14:45:29 +0200 Subject: [PATCH 06/32] Trigger buildkite on 1.12 --- .buildkite/run_tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/run_tests.yml b/.buildkite/run_tests.yml index ce4ddfd3..c4111fbf 100644 --- a/.buildkite/run_tests.yml +++ b/.buildkite/run_tests.yml @@ -5,6 +5,7 @@ steps: version: - "1.10" - "1.11" + - "1.12-nightly" plugins: - JuliaCI/julia#v1: version: "{{matrix.version}}" @@ -33,6 +34,7 @@ steps: version: - "1.10" - "1.11" + - "1.12-nightly" plugins: - JuliaCI/julia#v1: version: "{{matrix.version}}" From fd8226c84a8e97e55ef06b7bb2b4ad80e34e606f Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 14:49:44 +0200 Subject: [PATCH 07/32] Fixup --- ci/cscs-gh200.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 51c3bf36..4c8b5d80 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -5,11 +5,11 @@ unit_test: extends: .uenv-runner-daint-gh200 image: julia/25.5:v1 script: - - julia -e 'println("Instantiating project") - using Pkg + - julia -e 'println("Instantiating project"); + using Pkg; Pkg.develop(; path=pwd())' - - julia -e 'println("Running tests") - using Pkg + - julia -e 'println("Running tests"); + using Pkg; Pkg.test("JustPIC"; test_args=["--backend=CUDA"])' variables: WITH_UENV_VIEW: 'juliaup' @@ -22,6 +22,7 @@ ref_test: extends: .uenv-runner-daint-gh200 image: julia/25.5:v1 script: + - julia --project -e 'using Pkg; Pkg.instantiate()' - julia --project scripts/single_particle_advection_MPI.jl variables: WITH_UENV_VIEW: 'juliaup' From a96a984f63aaf25e34bae8f3bc374a15dd4b3648 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 14:52:41 +0200 Subject: [PATCH 08/32] Add missing deps --- ci/cscs-gh200.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 4c8b5d80..a0632864 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -23,6 +23,7 @@ ref_test: image: julia/25.5:v1 script: - julia --project -e 'using Pkg; Pkg.instantiate()' + - julia --project -e 'using Pkg; Pkg.add("CUDA")' - julia --project scripts/single_particle_advection_MPI.jl variables: WITH_UENV_VIEW: 'juliaup' From da33c65a9cb536b769f1721c681c6146e9ee71ad Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 15:14:45 +0200 Subject: [PATCH 09/32] Fixup ref test --- ci/cscs-gh200.yml | 2 +- scripts/single_particle_advection_MPI.jl | 5 +- scripts/single_particle_advection_MPI_ci.jl | 160 ++++++++++++++++++++ 3 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 scripts/single_particle_advection_MPI_ci.jl diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index a0632864..d5afb380 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -24,7 +24,7 @@ ref_test: script: - julia --project -e 'using Pkg; Pkg.instantiate()' - julia --project -e 'using Pkg; Pkg.add("CUDA")' - - julia --project scripts/single_particle_advection_MPI.jl + - julia --project scripts/single_particle_advection_MPI_ci.jl variables: WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 2 diff --git a/scripts/single_particle_advection_MPI.jl b/scripts/single_particle_advection_MPI.jl index c9786f29..9be94933 100644 --- a/scripts/single_particle_advection_MPI.jl +++ b/scripts/single_particle_advection_MPI.jl @@ -4,10 +4,7 @@ using JustPIC._2D # Threads is the default backend, # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") -# const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend - -using CUDA -const backend = JustPIC.CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend using GLMakie diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl new file mode 100644 index 00000000..4a999442 --- /dev/null +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -0,0 +1,160 @@ +using CUDA +using JustPIC, JustPIC._2D + +# Threads is the default backend, +# to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), +# and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") +const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend + +using ParallelStencil +@init_parallel_stencil(CUDA, Float64, 2) + +# using GLMakie + +@inline init_particle_fields_cellarrays(particles, ::Val{N}) where {N} = ntuple(_ -> @fill(0.0, size(particles.coords[1])..., celldims = (cellsize(particles.index))), Val(N)) + +function expand_range(x::AbstractRange) + dx = x[2] - x[1] + n = length(x) + x1, x2 = extrema(x) + xI = x1 - dx + xF = x2 + dx + return LinRange(xI, xF, n + 2) +end + +function expand_range(x::AbstractArray, dx) + x1, x2 = extrema(x) + xI = x1 - dx + xF = x2 + dx + return x = TA(vcat(xI, x, xF)) +end + +# Analytical flow solution +vx_stream(x, y) = 250 * sin(π * x) * cos(π * y) +vy_stream(x, y) = -250 * cos(π * x) * sin(π * y) +g(x) = Point2f( + vx_stream(x[1], x[2]), + vy_stream(x[1], x[2]) +) + +function main() + # Initialize particles ------------------------------- + nxcell, max_xcell, min_xcell = 24, 24, 18 + n = 32 + nx = ny = n - 1 + me, dims, = init_global_grid( + n - 1, n - 1, 0; + init_MPI = JustPIC.MPI.Initialized() ? false : true, + overlapx = 2, overlapy = 2 + ) + Lx = Ly = 1.0 + dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) + # nodal vertices + xvi = xv, yv = let + dummy = zeros(n, n) + xv = TA([x_g(i, dx, dummy) for i in axes(dummy, 1)]) + yv = TA([y_g(i, dx, dummy) for i in axes(dummy, 2)]) + xv, yv + end + # nodal centers + xci = xc, yc = let + dummy = zeros(nx, ny) + xc = @zeros(nx) + xc .= TA([x_g(i, dx, dummy) for i in axes(dummy, 1)]) + yc = TA([y_g(i, dx, dummy) for i in axes(dummy, 2)]) + xc, yc + end + + # staggered grid for the velocity components + grid_vx = xv, expand_range(yc, dy) + grid_vy = expand_range(xc, dx), yv + + particles = init_particle( + backend, nxcell, max_xcell, min_xcell, xv, yv, dxi..., nx, ny, me + ) + + # allocate particle field + particle_args = () + + # Cell fields ------------------------------- + Vx = TA([vx_stream(x, y) for x in grid_vx[1], y in grid_vx[2]]) + Vy = TA([vy_stream(x, y) for x in grid_vy[1], y in grid_vy[2]]) + V = Vx, Vy + + # time step + dt = min(dx / maximum(abs.(Vx)), dy / maximum(abs.(Vy))) + + nx_v = (size(particles.coords[1].data, 2)) * dims[1] + ny_v = (size(particles.coords[1].data, 3) - 2) * dims[2] + px_v = fill(NaN, nx_v, ny_v) + py_v = fill(NaN, nx_v, ny_v) + index_v = fill(false, nx_v, ny_v) + px_nohalo = fill(NaN, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) + py_nohalo = fill(NaN, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) + index_nohalo = fill(false, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) + + p = [(NaN, NaN)] + + # Advection test + niter = 150 + for iter in 1:niter + + # advect particles + advection!(particles, RungeKutta2(2 / 3), V, (grid_vx, grid_vy), dt) + # update halos + update_cell_halo!(particles.coords..., particle_args...) + update_cell_halo!(particles.index) + + # shuffle particles + move_particles!(particles, xvi, particle_args) + + # gather particle data - for plotting only + @views px_nohalo .= particles.coords[1].data[1, :, 2:(end - 1)] + @views py_nohalo .= particles.coords[2].data[1, :, 2:(end - 1)] + @views index_nohalo .= particles.index.data[1, :, 2:(end - 1)] + gather!(px_nohalo, px_v) + gather!(py_nohalo, py_v) + gather!(index_nohalo, index_v) + + if me == 0 + p_i = (px_v[index_v][1], py_v[index_v][1]) + push!(p, p_i) + end + + # if me == 0 && iter % 10 == 0 + # w = 0.504 + # offset = 0.5 - (w - 0.5) + # f, ax, = lines( + # [0, w, w, 0, 0], + # [0, 0, w, w, 0], + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0] .+ offset, + # [0, 0, w, w, 0], + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0] .+ offset, + # [0, 0, w, w, 0] .+ offset, + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0], + # [0, 0, w, w, 0] .+ offset, + # linewidth = 3 + # ) + # streamplot!(ax, g, LinRange(0, 1, 100), LinRange(0, 1, 100)) + # lines!(ax, p, color = :red) + # scatter!(ax, p[end], color = :black) + # save("figs/trajectory_MPI_$iter.png", f) + # end + end + + return finalize_global_grid() +end + +main() From a541beef1fddb9267ac423b9ac9c58494350055d Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 15:25:33 +0200 Subject: [PATCH 10/32] Fixup --- scripts/single_particle_advection_MPI_ci.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index 4a999442..49d063f3 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -1,6 +1,9 @@ using CUDA using JustPIC, JustPIC._2D +using ImplicitGlobalGrid +import MPI + # Threads is the default backend, # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") From 1d7b5bc81bb9d80210ab68aadb5cfaab712086b0 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 15:40:23 +0200 Subject: [PATCH 11/32] Fixup IGG call --- scripts/single_particle_advection_MPI_ci.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index 49d063f3..c176ab44 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -46,9 +46,8 @@ function main() n = 32 nx = ny = n - 1 me, dims, = init_global_grid( - n - 1, n - 1, 0; - init_MPI = JustPIC.MPI.Initialized() ? false : true, - overlapx = 2, overlapy = 2 + n - 1, n - 1, 1; + init_MPI = JustPIC.MPI.Initialized() ? false : true ) Lx = Ly = 1.0 dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) From f75b0badc70619e51ef5622df8c7f202eea2d1d4 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 15:51:14 +0200 Subject: [PATCH 12/32] More fixes --- scripts/single_particle_advection_MPI_ci.jl | 25 +++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index c176ab44..d9668ee2 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -1,4 +1,4 @@ -using CUDA +# using CUDA using JustPIC, JustPIC._2D using ImplicitGlobalGrid @@ -7,10 +7,12 @@ import MPI # Threads is the default backend, # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") -const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +# const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +const backend = JustPIC.CPUBackend using ParallelStencil -@init_parallel_stencil(CUDA, Float64, 2) +# @init_parallel_stencil(CUDA, Float64, 2) +@init_parallel_stencil(Threads, Float64, 2) # using GLMakie @@ -29,7 +31,7 @@ function expand_range(x::AbstractArray, dx) x1, x2 = extrema(x) xI = x1 - dx xF = x2 + dx - return x = TA(vcat(xI, x, xF)) + return x = TA(backend)(vcat(xI, x, xF)) end # Analytical flow solution @@ -54,16 +56,15 @@ function main() # nodal vertices xvi = xv, yv = let dummy = zeros(n, n) - xv = TA([x_g(i, dx, dummy) for i in axes(dummy, 1)]) - yv = TA([y_g(i, dx, dummy) for i in axes(dummy, 2)]) + xv = TA(backend)([x_g(i, dx, dummy) for i in axes(dummy, 1)]) + yv = TA(backend)([y_g(i, dx, dummy) for i in axes(dummy, 2)]) xv, yv end # nodal centers xci = xc, yc = let dummy = zeros(nx, ny) - xc = @zeros(nx) - xc .= TA([x_g(i, dx, dummy) for i in axes(dummy, 1)]) - yc = TA([y_g(i, dx, dummy) for i in axes(dummy, 2)]) + xc = TA(backend)([x_g(i, dx, dummy) for i in axes(dummy, 1)]) + yc = TA(backend)([y_g(i, dx, dummy) for i in axes(dummy, 2)]) xc, yc end @@ -71,7 +72,7 @@ function main() grid_vx = xv, expand_range(yc, dy) grid_vy = expand_range(xc, dx), yv - particles = init_particle( + particles = init_particles( backend, nxcell, max_xcell, min_xcell, xv, yv, dxi..., nx, ny, me ) @@ -79,8 +80,8 @@ function main() particle_args = () # Cell fields ------------------------------- - Vx = TA([vx_stream(x, y) for x in grid_vx[1], y in grid_vx[2]]) - Vy = TA([vy_stream(x, y) for x in grid_vy[1], y in grid_vy[2]]) + Vx = TA(backend)([vx_stream(x, y) for x in grid_vx[1], y in grid_vx[2]]) + Vy = TA(backend)([vy_stream(x, y) for x in grid_vy[1], y in grid_vy[2]]) V = Vx, Vy # time step From 001315df307118e3bfe73ff79f02a7486360a148 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 15:52:10 +0200 Subject: [PATCH 13/32] Fixup --- scripts/single_particle_advection_MPI_ci.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index d9668ee2..a3998d90 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -1,4 +1,4 @@ -# using CUDA +using CUDA using JustPIC, JustPIC._2D using ImplicitGlobalGrid @@ -7,12 +7,12 @@ import MPI # Threads is the default backend, # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") -# const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend -const backend = JustPIC.CPUBackend +const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +# const backend = JustPIC.CPUBackend using ParallelStencil -# @init_parallel_stencil(CUDA, Float64, 2) -@init_parallel_stencil(Threads, Float64, 2) +@init_parallel_stencil(CUDA, Float64, 2) +# @init_parallel_stencil(Threads, Float64, 2) # using GLMakie From 66510399bb0a9f45a5b7d2a906eaa9310373f7df Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 14 Aug 2025 17:45:01 +0200 Subject: [PATCH 14/32] fix init particles call --- scripts/single_particle_advection_MPI_ci.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index a3998d90..fe6a58b9 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -72,9 +72,7 @@ function main() grid_vx = xv, expand_range(yc, dy) grid_vy = expand_range(xc, dx), yv - particles = init_particles( - backend, nxcell, max_xcell, min_xcell, xv, yv, dxi..., nx, ny, me - ) + particles = init_particles(backend, nxcell, max_xcell, min_xcell, xvi...) # allocate particle field particle_args = () From 6a896635905046ba84ed022bb2cc2094610adff7 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 17:52:21 +0200 Subject: [PATCH 15/32] Fixup --- scripts/single_particle_advection_MPI_ci.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index fe6a58b9..66894d11 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -49,7 +49,8 @@ function main() nx = ny = n - 1 me, dims, = init_global_grid( n - 1, n - 1, 1; - init_MPI = JustPIC.MPI.Initialized() ? false : true + init_MPI = JustPIC.MPI.Initialized() ? false : true, + select_device = false ) Lx = Ly = 1.0 dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) From 7374f02bb474230fe56db33fc5ba3e01ba6fe9c1 Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 14 Aug 2025 18:02:50 +0200 Subject: [PATCH 16/32] allow scalar indexing --- scripts/single_particle_advection_MPI_ci.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index 66894d11..c4489b33 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -1,4 +1,5 @@ using CUDA +CUDA.allowscalar(true) # enable scalar operations on CUDA arrays using JustPIC, JustPIC._2D using ImplicitGlobalGrid From a6a8db029d91b5d48a137f97f87220c245939ce4 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:07:10 +0200 Subject: [PATCH 17/32] Allow scalar --- scripts/single_particle_advection_MPI_ci.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl index 66894d11..69cfb91f 100644 --- a/scripts/single_particle_advection_MPI_ci.jl +++ b/scripts/single_particle_advection_MPI_ci.jl @@ -10,6 +10,8 @@ import MPI const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend # const backend = JustPIC.CPUBackend +CUDA.allowscalar(true) + using ParallelStencil @init_parallel_stencil(CUDA, Float64, 2) # @init_parallel_stencil(Threads, Float64, 2) From 8b38f67b43b868be51438a1a4db6d3f30a28a145 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:08:27 +0200 Subject: [PATCH 18/32] try baremetal runner --- ci/cscs-gh200.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index d5afb380..6f04caab 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -18,15 +18,29 @@ unit_test: SLURM_GPUS_PER_TASK: 1 SLURM_TIMELIMIT: "00:30:00" +# ref_test: +# extends: .uenv-runner-daint-gh200 +# image: julia/25.5:v1 +# script: +# - julia --project -e 'using Pkg; Pkg.instantiate()' +# - julia --project -e 'using Pkg; Pkg.add("CUDA")' +# - julia --project scripts/single_particle_advection_MPI_ci.jl +# variables: +# WITH_UENV_VIEW: 'juliaup' +# SLURM_JOB_NUM_NODES: 2 +# SLURM_NTASKS_PER_NODE: 4 +# SLURM_GPUS_PER_TASK: 1 +# SLURM_TIMELIMIT: "00:30:00" + ref_test: - extends: .uenv-runner-daint-gh200 - image: julia/25.5:v1 + extends: .baremetal-runner-daint-gh200 script: - - julia --project -e 'using Pkg; Pkg.instantiate()' - - julia --project -e 'using Pkg; Pkg.add("CUDA")' - - julia --project scripts/single_particle_advection_MPI_ci.jl + - echo "Preparing the test environment (single rank)" + - srun -n 1 --uenv=julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' + - srun -n 1 --uenv=julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' + - echo "Running the reference test (multiple ranks)" + - srun --uenv=julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl variables: - WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 SLURM_GPUS_PER_TASK: 1 From afa9e826fe03968e1d815f2cd646d0ba347b7a07 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:11:03 +0200 Subject: [PATCH 19/32] Fixup --- ci/cscs-gh200.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 6f04caab..e51b565f 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -36,10 +36,10 @@ ref_test: extends: .baremetal-runner-daint-gh200 script: - echo "Preparing the test environment (single rank)" - - srun -n 1 --uenv=julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' - - srun -n 1 --uenv=julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' + - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' + - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' - echo "Running the reference test (multiple ranks)" - - srun --uenv=julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl + - srun --uenv julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl variables: SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 From 819f05822e2bea798b1878809a836bebfe94d062 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:13:34 +0200 Subject: [PATCH 20/32] Fixup --- ci/cscs-gh200.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index e51b565f..4cf32a8e 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -18,30 +18,30 @@ unit_test: SLURM_GPUS_PER_TASK: 1 SLURM_TIMELIMIT: "00:30:00" -# ref_test: -# extends: .uenv-runner-daint-gh200 -# image: julia/25.5:v1 -# script: -# - julia --project -e 'using Pkg; Pkg.instantiate()' -# - julia --project -e 'using Pkg; Pkg.add("CUDA")' -# - julia --project scripts/single_particle_advection_MPI_ci.jl -# variables: -# WITH_UENV_VIEW: 'juliaup' -# SLURM_JOB_NUM_NODES: 2 -# SLURM_NTASKS_PER_NODE: 4 -# SLURM_GPUS_PER_TASK: 1 -# SLURM_TIMELIMIT: "00:30:00" - ref_test: - extends: .baremetal-runner-daint-gh200 + extends: .uenv-runner-daint-gh200 + image: julia/25.5:v1 script: - - echo "Preparing the test environment (single rank)" - - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' - - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' - - echo "Running the reference test (multiple ranks)" - - srun --uenv julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl + - julia --project -e 'using Pkg; Pkg.instantiate()' + - julia --project -e 'using Pkg; Pkg.add("CUDA")' + - julia --project scripts/single_particle_advection_MPI_ci.jl variables: + WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 SLURM_GPUS_PER_TASK: 1 SLURM_TIMELIMIT: "00:30:00" + +# ref_test: +# extends: .baremetal-runner-daint-gh200 +# script: +# - echo "Preparing the test environment (single rank)" +# - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' +# - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' +# - echo "Running the reference test (multiple ranks)" +# - srun --uenv julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl +# variables: +# SLURM_JOB_NUM_NODES: 2 +# SLURM_NTASKS_PER_NODE: 4 +# SLURM_GPUS_PER_TASK: 1 +# SLURM_TIMELIMIT: "00:30:00" From 6a920c09f22c2ad954e9bd7ab7b3994bbd0c757f Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:21:51 +0200 Subject: [PATCH 21/32] Fixup --- ci/cscs-gh200.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 4cf32a8e..a2c78e6c 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -18,30 +18,30 @@ unit_test: SLURM_GPUS_PER_TASK: 1 SLURM_TIMELIMIT: "00:30:00" -ref_test: - extends: .uenv-runner-daint-gh200 - image: julia/25.5:v1 - script: - - julia --project -e 'using Pkg; Pkg.instantiate()' - - julia --project -e 'using Pkg; Pkg.add("CUDA")' - - julia --project scripts/single_particle_advection_MPI_ci.jl - variables: - WITH_UENV_VIEW: 'juliaup' - SLURM_JOB_NUM_NODES: 2 - SLURM_NTASKS_PER_NODE: 4 - SLURM_GPUS_PER_TASK: 1 - SLURM_TIMELIMIT: "00:30:00" - # ref_test: -# extends: .baremetal-runner-daint-gh200 +# extends: .uenv-runner-daint-gh200 +# image: julia/25.5:v1 # script: -# - echo "Preparing the test environment (single rank)" -# - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.instantiate()' -# - srun -n 1 --uenv julia/25.5:v1 julia --project -e 'using Pkg; Pkg.add("CUDA")' -# - echo "Running the reference test (multiple ranks)" -# - srun --uenv julia/25.5:v1 julia --project scripts/single_particle_advection_MPI_ci.jl +# - julia --project -e 'using Pkg; Pkg.instantiate()' +# - julia --project -e 'using Pkg; Pkg.add("CUDA")' +# - julia --project scripts/single_particle_advection_MPI_ci.jl # variables: +# WITH_UENV_VIEW: 'juliaup' # SLURM_JOB_NUM_NODES: 2 # SLURM_NTASKS_PER_NODE: 4 # SLURM_GPUS_PER_TASK: 1 # SLURM_TIMELIMIT: "00:30:00" + +ref_test: + extends: .baremetal-runner-daint-gh200 + script: + - echo "Preparing the test environment (single rank)" + - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.instantiate()' + - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.add("CUDA")' + - echo "Running the reference test (multiple ranks)" + - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/single_particle_advection_MPI_ci.jl + variables: + SLURM_JOB_NUM_NODES: 2 + SLURM_NTASKS_PER_NODE: 4 + SLURM_GPUS_PER_TASK: 1 + SLURM_TIMELIMIT: "00:30:00" From 2eda7ff1fcf673af137e94f6451ebb987d991427 Mon Sep 17 00:00:00 2001 From: Albert de Montserrat Date: Thu, 14 Aug 2025 18:23:34 +0200 Subject: [PATCH 22/32] update 2D MPI advection --- scripts/temperature_advection_MPI.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/temperature_advection_MPI.jl b/scripts/temperature_advection_MPI.jl index 623eaae8..a64ba91d 100644 --- a/scripts/temperature_advection_MPI.jl +++ b/scripts/temperature_advection_MPI.jl @@ -7,7 +7,7 @@ using JustPIC, JustPIC._2D # to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), # and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") # const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend -const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +# const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend using GLMakie using ImplicitGlobalGrid @@ -87,9 +87,9 @@ function main() advection!(particles, RungeKutta2(), V, (grid_vx, grid_vy), dt) # update halos - update_cell_halo!(particles.coords...) - update_cell_halo!(particle_args...) - update_cell_halo!(particles.index) + update_halo!(particles.coords...) + update_halo!(particle_args...) + update_halo!(particles.index) # shuffle particles move_particles!(particles, xvi, particle_args) # interpolate T from particle to grid From b356294a4192c5ffc24c089b41bdfbcb63b4309a Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:41:25 +0200 Subject: [PATCH 23/32] New try --- ci/cscs-gh200.yml | 3 +- scripts/temperature_advection_MPI_ci.jl | 153 ++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 scripts/temperature_advection_MPI_ci.jl diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index a2c78e6c..9f97c943 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -39,7 +39,8 @@ ref_test: - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.instantiate()' - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.add("CUDA")' - echo "Running the reference test (multiple ranks)" - - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/single_particle_advection_MPI_ci.jl + # - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/single_particle_advection_MPI_ci.jl + - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/temperature_advection_MPI_ci variables: SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 diff --git a/scripts/temperature_advection_MPI_ci.jl b/scripts/temperature_advection_MPI_ci.jl new file mode 100644 index 00000000..baa6733e --- /dev/null +++ b/scripts/temperature_advection_MPI_ci.jl @@ -0,0 +1,153 @@ +using CUDA +# import Pkg +# Pkg.resolve(); Pkg.update() +using JustPIC, JustPIC._2D + +# Threads is the default backend, +# to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), +# and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") +# const backend = JustPIC.CPUBackend # Options: CPUBackend, CUDABackend, AMDGPUBackend +const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend + +# using GLMakie +using ImplicitGlobalGrid +import MPI + +# Analytical flow solution +vx_stream(x, y) = 250 * sin(π * x) * cos(π * y) +vy_stream(x, y) = -250 * cos(π * x) * sin(π * y) +g(x) = Point2f( + vx_stream(x[1], x[2]), + vy_stream(x[1], x[2]) +) + +function expand_range(x::AbstractRange) + dx = x[2] - x[1] + n = length(x) + x1, x2 = extrema(x) + xI = x1 - dx + xF = x2 + dx + return LinRange(xI, xF, n + 2) +end + +function main() + # Initialize particles ------------------------------- + nxcell, max_xcell, min_xcell = 24, 40, 1 + n = 64 + nx = ny = n - 1 + me, dims, = init_global_grid( + n - 1, n - 1, 1; + init_MPI = MPI.Initialized() ? false : true, + select_device = false + ) + Lx = Ly = 1.0 + dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) + # nodal vertices + xvi = xv, yv = let + dummy = zeros(n, n) + xv = [x_g(i, dx, dummy) for i in axes(dummy, 1)] + yv = [y_g(i, dy, dummy) for i in axes(dummy, 2)] + LinRange(first(xv), last(xv), n), LinRange(first(yv), last(yv), n) + end + # nodal centers + xci = xc, yc = let + dummy = zeros(nx, ny) + xc = [x_g(i, dx, dummy) for i in axes(dummy, 1)] + yc = [y_g(i, dy, dummy) for i in axes(dummy, 2)] + LinRange(first(xc), last(xc), n), LinRange(first(yc), last(yc), n) + end + + # staggered grid for the velocity components + grid_vx = xv, expand_range(yc) + grid_vy = expand_range(xc), yv + + particles = init_particles( + backend, nxcell, max_xcell, min_xcell, xvi... + ) + + # Cell fields ------------------------------- + Vx = TA(backend)([vx_stream(x, y) for x in grid_vx[1], y in grid_vx[2]]) + Vy = TA(backend)([vy_stream(x, y) for x in grid_vy[1], y in grid_vy[2]]) + T = TA(backend)([y for x in xv, y in yv]) + T0 = deepcopy(T) + V = Vx, Vy + + nx_v = (size(T, 1) - 2) * dims[1] + ny_v = (size(T, 2) - 2) * dims[2] + T_v = zeros(nx_v, ny_v) + T_nohalo = TA(backend)(zeros(size(T) .- 2)) + + dt = mapreduce(x -> x[1] / MPI.Allreduce(maximum(abs.(x[2])), MPI.MAX, MPI.COMM_WORLD), min, zip(dxi, V)) / 2 + + # Advection test + particle_args = pT, = init_cell_arrays(particles, Val(1)) + grid2particle!(pT, xvi, T, particles) + + niter = 250 + for iter in 1:niter + me == 0 && @show iter + + # advect particles + advection!(particles, RungeKutta2(), V, (grid_vx, grid_vy), dt) + + # update halos + update_cell_halo!(particles.coords...) + update_cell_halo!(particle_args...) + update_cell_halo!(particles.index) + # shuffle particles + move_particles!(particles, xvi, particle_args) + # interpolate T from particle to grid + particle2grid!(T, pT, xvi, particles) + + @views T_nohalo .= T[2:(end - 1), 2:(end - 1)] + gather!(Array(T_nohalo), T_v) + + # if me == 0 && iter % 1 == 0 + # x_global = range(0, Lx, length = size(T_v, 1)) + # y_global = range(0, Ly, length = size(T_v, 2)) + # f, ax, = heatmap(x_global, y_global, T_v) + # w = 0.504 + # offset = 0.5 - (w - 0.5) + # lines!( + # ax, + # [0, w, w, 0, 0], + # [0, 0, w, w, 0], + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0] .+ offset, + # [0, 0, w, w, 0], + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0] .+ offset, + # [0, 0, w, w, 0] .+ offset, + # linewidth = 3 + # ) + # lines!( + # ax, + # [0, w, w, 0, 0], + # [0, 0, w, w, 0] .+ offset, + # linewidth = 3 + # ) + + # save("figs/T_MPI_$iter.png", f) + # end + + # px = particles.coords[1].data[:] + # py = particles.coords[2].data[:] + # idx = particles.index.data[:] + # f = scatter(px[idx], py[idx], color=:black) + # save("figs/particles_$(iter)_$(me).png", f) + end + + # f, ax, = heatmap(xvi..., T, colormap=:batlow) + # streamplot!(ax, g, xvi...) + # f + return finalize_global_grid() + +end + +main() From 1500f050c3455b650d20c97bd17aadbf1cf3ce00 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:44:51 +0200 Subject: [PATCH 24/32] Up --- ci/cscs-gh200.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 9f97c943..3c87aa9f 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -40,7 +40,7 @@ ref_test: - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.add("CUDA")' - echo "Running the reference test (multiple ranks)" # - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/single_particle_advection_MPI_ci.jl - - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/temperature_advection_MPI_ci + - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/temperature_advection_MPI_ci.jl variables: SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 From 6016e34ba51b91c607c5429adf6734d7e0a198a8 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:49:45 +0200 Subject: [PATCH 25/32] Cleanup --- ci/cscs-gh200.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 3c87aa9f..85316b03 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -18,20 +18,6 @@ unit_test: SLURM_GPUS_PER_TASK: 1 SLURM_TIMELIMIT: "00:30:00" -# ref_test: -# extends: .uenv-runner-daint-gh200 -# image: julia/25.5:v1 -# script: -# - julia --project -e 'using Pkg; Pkg.instantiate()' -# - julia --project -e 'using Pkg; Pkg.add("CUDA")' -# - julia --project scripts/single_particle_advection_MPI_ci.jl -# variables: -# WITH_UENV_VIEW: 'juliaup' -# SLURM_JOB_NUM_NODES: 2 -# SLURM_NTASKS_PER_NODE: 4 -# SLURM_GPUS_PER_TASK: 1 -# SLURM_TIMELIMIT: "00:30:00" - ref_test: extends: .baremetal-runner-daint-gh200 script: From 504f439abf7638b86ad284d961373e36dd71f5ef Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:50:20 +0200 Subject: [PATCH 26/32] Cleanup --- scripts/single_particle_advection_MPI_ci.jl | 165 -------------------- 1 file changed, 165 deletions(-) delete mode 100644 scripts/single_particle_advection_MPI_ci.jl diff --git a/scripts/single_particle_advection_MPI_ci.jl b/scripts/single_particle_advection_MPI_ci.jl deleted file mode 100644 index bf27382f..00000000 --- a/scripts/single_particle_advection_MPI_ci.jl +++ /dev/null @@ -1,165 +0,0 @@ -using CUDA -CUDA.allowscalar(true) # enable scalar operations on CUDA arrays -using JustPIC, JustPIC._2D - -using ImplicitGlobalGrid -import MPI - -# Threads is the default backend, -# to run on a CUDA GPU load CUDA.jl (i.e. "using CUDA"), -# and to run on an AMD GPU load AMDGPU.jl (i.e. "using AMDGPU") -const backend = CUDABackend # Options: CPUBackend, CUDABackend, AMDGPUBackend -# const backend = JustPIC.CPUBackend - -CUDA.allowscalar(true) - -using ParallelStencil -@init_parallel_stencil(CUDA, Float64, 2) -# @init_parallel_stencil(Threads, Float64, 2) - -# using GLMakie - -@inline init_particle_fields_cellarrays(particles, ::Val{N}) where {N} = ntuple(_ -> @fill(0.0, size(particles.coords[1])..., celldims = (cellsize(particles.index))), Val(N)) - -function expand_range(x::AbstractRange) - dx = x[2] - x[1] - n = length(x) - x1, x2 = extrema(x) - xI = x1 - dx - xF = x2 + dx - return LinRange(xI, xF, n + 2) -end - -function expand_range(x::AbstractArray, dx) - x1, x2 = extrema(x) - xI = x1 - dx - xF = x2 + dx - return x = TA(backend)(vcat(xI, x, xF)) -end - -# Analytical flow solution -vx_stream(x, y) = 250 * sin(π * x) * cos(π * y) -vy_stream(x, y) = -250 * cos(π * x) * sin(π * y) -g(x) = Point2f( - vx_stream(x[1], x[2]), - vy_stream(x[1], x[2]) -) - -function main() - # Initialize particles ------------------------------- - nxcell, max_xcell, min_xcell = 24, 24, 18 - n = 32 - nx = ny = n - 1 - me, dims, = init_global_grid( - n - 1, n - 1, 1; - init_MPI = JustPIC.MPI.Initialized() ? false : true, - select_device = false - ) - Lx = Ly = 1.0 - dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) - # nodal vertices - xvi = xv, yv = let - dummy = zeros(n, n) - xv = TA(backend)([x_g(i, dx, dummy) for i in axes(dummy, 1)]) - yv = TA(backend)([y_g(i, dx, dummy) for i in axes(dummy, 2)]) - xv, yv - end - # nodal centers - xci = xc, yc = let - dummy = zeros(nx, ny) - xc = TA(backend)([x_g(i, dx, dummy) for i in axes(dummy, 1)]) - yc = TA(backend)([y_g(i, dx, dummy) for i in axes(dummy, 2)]) - xc, yc - end - - # staggered grid for the velocity components - grid_vx = xv, expand_range(yc, dy) - grid_vy = expand_range(xc, dx), yv - - particles = init_particles(backend, nxcell, max_xcell, min_xcell, xvi...) - - # allocate particle field - particle_args = () - - # Cell fields ------------------------------- - Vx = TA(backend)([vx_stream(x, y) for x in grid_vx[1], y in grid_vx[2]]) - Vy = TA(backend)([vy_stream(x, y) for x in grid_vy[1], y in grid_vy[2]]) - V = Vx, Vy - - # time step - dt = min(dx / maximum(abs.(Vx)), dy / maximum(abs.(Vy))) - - nx_v = (size(particles.coords[1].data, 2)) * dims[1] - ny_v = (size(particles.coords[1].data, 3) - 2) * dims[2] - px_v = fill(NaN, nx_v, ny_v) - py_v = fill(NaN, nx_v, ny_v) - index_v = fill(false, nx_v, ny_v) - px_nohalo = fill(NaN, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) - py_nohalo = fill(NaN, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) - index_nohalo = fill(false, size(particles.coords[1].data, 2), size(particles.coords[1].data, 3) - 2) - - p = [(NaN, NaN)] - - # Advection test - niter = 150 - for iter in 1:niter - - # advect particles - advection!(particles, RungeKutta2(2 / 3), V, (grid_vx, grid_vy), dt) - # update halos - update_cell_halo!(particles.coords..., particle_args...) - update_cell_halo!(particles.index) - - # shuffle particles - move_particles!(particles, xvi, particle_args) - - # gather particle data - for plotting only - @views px_nohalo .= particles.coords[1].data[1, :, 2:(end - 1)] - @views py_nohalo .= particles.coords[2].data[1, :, 2:(end - 1)] - @views index_nohalo .= particles.index.data[1, :, 2:(end - 1)] - gather!(px_nohalo, px_v) - gather!(py_nohalo, py_v) - gather!(index_nohalo, index_v) - - if me == 0 - p_i = (px_v[index_v][1], py_v[index_v][1]) - push!(p, p_i) - end - - # if me == 0 && iter % 10 == 0 - # w = 0.504 - # offset = 0.5 - (w - 0.5) - # f, ax, = lines( - # [0, w, w, 0, 0], - # [0, 0, w, w, 0], - # linewidth = 3 - # ) - # lines!( - # ax, - # [0, w, w, 0, 0] .+ offset, - # [0, 0, w, w, 0], - # linewidth = 3 - # ) - # lines!( - # ax, - # [0, w, w, 0, 0] .+ offset, - # [0, 0, w, w, 0] .+ offset, - # linewidth = 3 - # ) - # lines!( - # ax, - # [0, w, w, 0, 0], - # [0, 0, w, w, 0] .+ offset, - # linewidth = 3 - # ) - # streamplot!(ax, g, LinRange(0, 1, 100), LinRange(0, 1, 100)) - # lines!(ax, p, color = :red) - # scatter!(ax, p[end], color = :black) - # save("figs/trajectory_MPI_$iter.png", f) - # end - end - - return finalize_global_grid() -end - -main() From 39db6f23b907184dcca189249690410bc5fb50fe Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 18:50:35 +0200 Subject: [PATCH 27/32] Cleanup --- ci/cscs-gh200.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 85316b03..4c428bc9 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -25,7 +25,6 @@ ref_test: - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.instantiate()' - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.add("CUDA")' - echo "Running the reference test (multiple ranks)" - # - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/single_particle_advection_MPI_ci.jl - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/temperature_advection_MPI_ci.jl variables: SLURM_JOB_NUM_NODES: 2 From 96987972ad3e1f3af3c0167dd67cc5a9f89c5944 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 19:12:45 +0200 Subject: [PATCH 28/32] Format --- scripts/temperature_advection_MPI_ci.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/temperature_advection_MPI_ci.jl b/scripts/temperature_advection_MPI_ci.jl index baa6733e..8caa13d8 100644 --- a/scripts/temperature_advection_MPI_ci.jl +++ b/scripts/temperature_advection_MPI_ci.jl @@ -39,7 +39,7 @@ function main() n - 1, n - 1, 1; init_MPI = MPI.Initialized() ? false : true, select_device = false - ) + ) Lx = Ly = 1.0 dxi = dx, dy = Lx / (nx_g() - 1), Ly / (ny_g() - 1) # nodal vertices From f6d1da60a5d229d9ae050a2ebe0db93f25d628b5 Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Thu, 14 Aug 2025 19:24:33 +0200 Subject: [PATCH 29/32] AMDGPU fix trial --- ext/JustPICAMDGPUExt.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/JustPICAMDGPUExt.jl b/ext/JustPICAMDGPUExt.jl index cbd44e81..d9e8f991 100644 --- a/ext/JustPICAMDGPUExt.jl +++ b/ext/JustPICAMDGPUExt.jl @@ -269,8 +269,8 @@ module _2D end function JustPIC._2D.semilagrangian_advection!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, @@ -282,8 +282,8 @@ module _2D end function JustPIC._2D.semilagrangian_advection_LinP!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, @@ -295,8 +295,8 @@ module _2D end function JustPIC._2D.semilagrangian_advection_MQS!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, @@ -703,8 +703,8 @@ module _3D end function JustPIC._3D.semilagrangian_advection!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, @@ -716,8 +716,8 @@ module _3D end function JustPIC._3D.semilagrangian_advection_LinP!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, @@ -729,8 +729,8 @@ module _3D end function JustPIC._3D.semilagrangian_advection_MQS!( - F::Union{ROCArrays, NTuple{NF, ROCArrays}}, - F0::Union{ROCArrays, NTuple{NF, ROCArrays}}, + F::Union{ROCArray, NTuple{NF, ROCArray}}, + F0::Union{ROCArray, NTuple{NF, ROCArray}}, method::AbstractAdvectionIntegrator, V, grid_vi::NTuple{N, NTuple{N, T}}, From 0e296c22e52da3c40ccc0e20df24d98029fc2d2d Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Fri, 15 Aug 2025 09:27:36 +0200 Subject: [PATCH 30/32] Fixup project --- ci/cscs-gh200.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 4c428bc9..33f24dc9 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -7,9 +7,10 @@ unit_test: script: - julia -e 'println("Instantiating project"); using Pkg; - Pkg.develop(; path=pwd())' + Pkg.activate(pwd())' - julia -e 'println("Running tests"); using Pkg; + Pkg.activate(pwd()); Pkg.test("JustPIC"; test_args=["--backend=CUDA"])' variables: WITH_UENV_VIEW: 'juliaup' @@ -22,10 +23,10 @@ ref_test: extends: .baremetal-runner-daint-gh200 script: - echo "Preparing the test environment (single rank)" - - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.instantiate()' - - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project -e 'using Pkg; Pkg.add("CUDA")' + - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project=. -e 'using Pkg; Pkg.instantiate()' + - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project=. -e 'using Pkg; Pkg.add("CUDA")' - echo "Running the reference test (multiple ranks)" - - srun --uenv julia/25.5:v1 --view=juliaup julia --project scripts/temperature_advection_MPI_ci.jl + - srun --uenv julia/25.5:v1 --view=juliaup julia --project=. scripts/temperature_advection_MPI_ci.jl variables: SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS_PER_NODE: 4 From bca90e6c68a2ec1e8d0d7429a3f87b4115919e5b Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Fri, 15 Aug 2025 09:29:47 +0200 Subject: [PATCH 31/32] Fixup format --- ci/cscs-gh200.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/ci/cscs-gh200.yml b/ci/cscs-gh200.yml index 33f24dc9..c53c2a98 100644 --- a/ci/cscs-gh200.yml +++ b/ci/cscs-gh200.yml @@ -5,13 +5,16 @@ unit_test: extends: .uenv-runner-daint-gh200 image: julia/25.5:v1 script: - - julia -e 'println("Instantiating project"); - using Pkg; - Pkg.activate(pwd())' - - julia -e 'println("Running tests"); - using Pkg; - Pkg.activate(pwd()); - Pkg.test("JustPIC"; test_args=["--backend=CUDA"])' + - export MPICH_GPU_SUPPORT_ENABLED=1 + - export IGG_CUDAAWARE_MPI=1 # IGG + - export JULIA_CUDA_USE_COMPAT=false # IGG + - julia -e 'println("Instantiating project"); + using Pkg; + Pkg.activate(pwd())' + - julia -e 'println("Running tests"); + using Pkg; + Pkg.activate(pwd()); + Pkg.test("JustPIC"; test_args=["--backend=CUDA"])' variables: WITH_UENV_VIEW: 'juliaup' SLURM_JOB_NUM_NODES: 1 @@ -22,6 +25,9 @@ unit_test: ref_test: extends: .baremetal-runner-daint-gh200 script: + - export MPICH_GPU_SUPPORT_ENABLED=1 + - export IGG_CUDAAWARE_MPI=1 # IGG + - export JULIA_CUDA_USE_COMPAT=false # IGG - echo "Preparing the test environment (single rank)" - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project=. -e 'using Pkg; Pkg.instantiate()' - srun -n 1 --uenv julia/25.5:v1 --view=juliaup julia --project=. -e 'using Pkg; Pkg.add("CUDA")' From ca1fb3f6cdcd29e148b361c4141857dee5ad12ec Mon Sep 17 00:00:00 2001 From: Ludovic Raess Date: Tue, 19 Aug 2025 09:05:48 +0200 Subject: [PATCH 32/32] Add badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d527bc55..8b741169 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![DOI](https://zenodo.org/badge/507905159.svg)](https://zenodo.org/doi/10.5281/zenodo.10212675) [![CPU UnitTests](https://github.com/JuliaGeodynamics/JustPIC.jl/actions/workflows/UnitTests.yml/badge.svg)](https://github.com/JuliaGeodynamics/JustPIC.jl/actions/workflows/UnitTests.yml) [![GPU UnitTests](https://badge.buildkite.com/bb05ed7ef3b43f843a5ba4a976c27a724064d67955193accea.svg)](https://buildkite.com/julialang/justpic-dot-jl) +[![CSCS UnitTests](https://gitlab.com/cscs-ci/ci-testing/webhook-ci/mirrors/6264856887055800/8444213376739374/badges/main/pipeline.svg?ignore_skipped=true)](https://gitlab.com/cscs-ci/ci-testing/webhook-ci/mirrors/6264856887055800/8444213376739374/-/pipelines) [![codecov](https://codecov.io/gh/JuliaGeodynamics/JustPIC.jl/graph/badge.svg?token=PN0AJZXK13)](https://codecov.io/gh/JuliaGeodynamics/JustPIC.jl) [![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)