Skip to content

Conversion to non-managed GPU memory #5

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
drreynolds wants to merge 17 commits into
base: main
Choose a base branch
from
Open

Conversion to non-managed GPU memory #5

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
8091388
Initial work toward moving reaction rate tables to non-managed memory…
drreynolds May 31, 2022
e31816d
Converted code to allocate separate host/device reaction rate structu…
drreynolds Jun 13, 2022
586106f
Streamlined dual host/device reaction network data structure implemen…
drreynolds Jun 14, 2022
7e4eb44
Changed allocation of dual host/device reaction rate structure to gen…
drreynolds Jun 14, 2022
a080ca6
Completed transition to non-managed memory for all physical test prob…
drreynolds Jun 16, 2022
1e3c63b
Minor fix from last commit
drreynolds Jun 16, 2022
294c4d6
Minor cleanup for HIP-based GPUs
drreynolds Jun 16, 2022
106b194
Reverted multirate_chem_hydro_main to use the 'managed' RAJA vector (…
drreynolds Jun 16, 2022
d373646
Updates to enable HIP-based build
drreynolds Jun 20, 2022
742eca7
Moved declaration of EulerData structure until after MPI_Init is called
drreynolds Jun 21, 2022
8908b94
Reverted imex_chem_hydro_main.cpp to use managed RAJA vector (as alre…
drreynolds Jun 21, 2022
d712ffa
Merge branch 'feature/non-managed' of github.com:sundials-codes/sundi…
drreynolds Jun 21, 2022
c874229
Merged main into feature/non-managed branch (needs testing)
drreynolds Dec 23, 2022
cc73e63
Finished updating non-managed branch with changes from main. Tested …
drreynolds Jan 5, 2023
e2efadb
Added HIP specifics to CMakeLists.txt
drreynolds Apr 11, 2023
0ac28ea
Updated build setup for Crusher
drreynolds Apr 12, 2023
b294087
Merge updates from main into feature/non-managed in preparation for a…
drreynolds May 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,15 @@ set(RAJA_BACKEND "SERIAL"
CACHE STRING "RAJA Backend to use: SERIAL, OPENMP, CUDA, HIP")
set_property(CACHE RAJA_BACKEND PROPERTY STRINGS "SERIAL;OPENMP;CUDA;HIP")

if(RAJA_BACKEND MATCHES "HIP")
find_package(HIP)
enable_language(HIP)
show_language_flags(HIP)
set(AMDGPU_TARGETS "gfx90a" CACHE STRING "HIP Architecture")
set(CMAKE_HIP_STANDARD ${CMAKE_CXX_STANDARD})
endif()


set(RAJA_ROOT "$ENV{RAJA_ROOT}"
CACHE PATH "Root directory of RAJA install")

Expand All @@ -83,8 +92,8 @@ message(STATUS "Found RAJA: ${RAJA_DIR}")
mark_as_advanced(FORCE RAJA_DIR)

if(RAJA_BACKEND MATCHES "CUDA")
enable_language(CUDA)
show_language_flags(CUDA)
enable_language(CUDA)
show_language_flags(CUDA)
set(CMAKE_CUDA_ARCHITECTURES "70" CACHE STRING "CUDA Architecture")
set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
endif()
Expand Down
364 changes: 364 additions & 0 deletions spack/spack-crusher.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,364 @@
spack:
repos: []
view: false
concretizer:
reuse: false
unify: false
modules:
default:
'enable:': [lmod]
lmod:
defaults: [[email protected]]
core_compilers: [[email protected]]
verbose: true
hash_length: 0
hierarchy: [mpi]
projections: {}
core_specs: []
all:
autoload: direct
environment:
set:
${PACKAGE}_ROOT: ${PREFIX}
suffixes:
+rocm: rocm
py-warpx:
suffixes:
^warpx dims=2: dims2
^warpx dims=3: dims3
^warpx dims=rz: dimsRZ
exclude_implicits: true
include: [cray-mpich]
prefix_inspections:
lib64: [LD_LIBRARY_PATH]
lib: [LD_LIBRARY_PATH]
compilers:
- compiler:
spec: [email protected]
paths:
cc: cc
cxx: CC
f77: ftn
fc: ftn
flags: {}
operating_system: sles15
target: any
modules:
- PrgEnv-amd/8.3.3
- amd/5.4.3
- xpmem
- craype-x86-trento
- libfabric
- cray-pmi/6.1.3
environment:
set:
RFE_811452_DISABLE: '1'
append_path:
PKG_CONFIG_PATH: /opt/cray/xpmem/2.4.4-2.3_11.2__gff0e1d9.shasta/lib64/pkgconfig
LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
prepend_path:
LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.3/lib
LIBRARY_PATH: /opt/rocm-5.4.3/lib:/opt/rocm-5.4.3/lib64
extra_rpaths:
- /opt/rocm-5.4.3/lib
- /opt/rocm-5.4.3/lib64
- /opt/cray/pe/gcc-libs
- /opt/cray/gcc-libs
- compiler:
spec: [email protected]
paths:
cc: gcc
cxx: g++
f77: gfortran
fc: gfortran
flags: {}
operating_system: sles15
target: any
modules:
- PrgEnv-gnu
- gcc/11.2.0
environment:
set:
RFE_811452_DISABLE: '1'
append_path:
LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.0.0/lib64
prepend_path:
LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.3/lib
extra_rpaths:
- /opt/cray/pe/gcc-libs
- /opt/cray/gcc-libs

packages:
all:
require: '%[email protected]'
compiler: [[email protected]]
providers:
blas: [openblas]
lapack: [openblas]
mpi: [cray-mpich]
scalapack: [netlib-scalapack]
target: [sles15-zen3]
variants: +mpi
binutils:
variants: +ld +gold +headers +libiberty ~nls
elfutils:
variants: +bzip2 ~nls +xz
hdf5:
variants: +fortran +hl +shared
libfabric:
buildable: false
externals:
- spec: [email protected]
modules:
- libfabric/1.15.0.0

# ROCm 5.4.3
libunwind:
variants: +pic +xz
ncurses:
require: '@6.3 +termlib'
openblas:
variants: threads=openmp
externals:
- spec: [email protected] threads=openmp
modules:
- openblas/0.3.17-omp
python:
require: '@3.7.15'
trilinos:
variants: +amesos +amesos2 +anasazi +aztec +belos +boost +epetra +epetraext
+ifpack +ifpack2 +intrepid +intrepid2 +isorropia +kokkos +ml +minitensor +muelu
+nox +piro +phalanx +rol +rythmos +sacado +stk +shards +shylu +stokhos +stratimikos
+teko +tempus +tpetra +trilinoscouplings +zoltan +zoltan2 +superlu-dist gotype=long_long
xz:
variants: +pic
mesa:
version: [21.3.8]
unzip:
require: '%gcc'
petsc:
variants: +batch
parallel-netcdf:
variants: +cxx +fortran +fpic
netlib-scalapack:
variants: +fpic
netcdf-c:
variants: ~hdf4 +mpi +parallel-netcdf +shared
fftw:
variants: precision=float,double,long_double
darshan-runtime:
variants: scheduler=slurm
cray-mpich:
buildable: false
externals:
- spec: [email protected] %[email protected]
prefix: /opt/cray/pe/mpich/8.1.23/ofi/amd/5.0
modules:
- cray-mpich/8.1.23
- libfabric/1.15.0.0
cmake:
externals:
- spec: [email protected]
modules: [cmake/3.23.2]
comgr:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hip-rocclr:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/hip
modules: [rocm/5.4.3]
hipblas:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hipcub:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hipfft:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hipsparse:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
miopen-hip:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
miopengemm:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rccl:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocblas:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocfft:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-clang-ocl:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-cmake:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-dbgapi:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-debug-agent:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-device-libs:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-gdb:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
rocm-opencl:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/opencl
modules: [rocm/5.4.3]
rocm-smi-lib:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hip:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules:
- craype-accel-amd-gfx90a
- rocm/5.4.3
extra_attributes:
compilers:
c: CC
c++: CC
hip: CC
environment:
set:
MPICH_GPU_SUPPORT_ENABLED: 1
llvm-amdgpu:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/llvm
modules: [rocm/5.4.3]
extra_attributes:
compilers:
c: /opt/rocm-5.4.3/llvm/bin/clang++
cxx: /opt/rocm-5.4.3/llvm/bin/clang++
hsakmt-roct:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
hsa-rocr-dev:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/
modules: [rocm/5.4.3]
extra_atributes:
compilers:
c: /opt/rocm-5.4.3/llvm/bin/clang++
cxx: /opt/rocm-5.4.3/llvm/bin/clang++
roctracer-dev-api:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3/roctracer
modules: [rocm/5.4.3]
rocprim:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules: [rocm/5.4.3]
rocrand:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules: [rocm/5.4.3]
rocsolver:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules: [rocm/5.4.3]
rocsparse:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules: [rocm/5.4.3]
rocthrust:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules: [rocm/5.4.3]
rocprofiler-dev:
buildable: false
externals:
- spec: [email protected]
prefix: /opt/rocm-5.4.3
modules:
- rocm/5.4.3
specs:
- [email protected]%[email protected]+magma+mpi+raja+rocm amdgpu_target=gfx90a
^magma+rocm amdgpu_target=gfx90a ^[email protected]+rocm amdgpu_target=gfx90a
Loading