llnl · ldowen · Aug 20, 2025 · Aug 21, 2025 · Aug 21, 2025 · Aug 25, 2025
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -56,6 +56,12 @@ Notable changes include:
       * Added std::span (boost::span until we move to C++20) version of view classes for Field and FieldList. This allows us to avoid complicated external systems like CHAI::ManagedArray for unified memory systems.
         * New CMake configuration variable SPHERAL_UNIFIED_MEMORY switches between using span or ManagedArray in the view classes (default to OFF, which means ManagedArray).
       * Converted Geometry Tensor types to be entirely inlined and host/device compliant.
+      * ArtificialViscosity has been refactored for use on the GPU.
+        * ArtificialViscosity is now ArtificialViscosityView.
+        * ArtificialViscosityHandle is now ArtificialViscosity.
+        * Both inherit from ArtificialViscosityBase.
+        * FiniteVolumeViscosity, MonGViscosity, LimitedMonGViscosity, and TensorMonGViscosity are split into a value and view class.
+      * FieldNames are inlined for easier use on device.
       * Silo python wrappers are now installed and accessible through the Spheral virtual python environment but currently unused.
 
   * Bug fixes
@@ -87,6 +93,7 @@ Notable changes include:
         `add_compile_definition` or `add_compile_options`.
       * Compiler flags are set for HIP or CXX depending on the configuration.
       * Update BLT to version 0.7.1.
+      * Added `CXX_LINK_FLAGS`.
     * Target exporting is now being tested in the CI on the RZ.
     * Updating boost function calls to std library implementations where possible.
     * Switched the CZ CI to use Dane instead of Ruby.

diff --git a/cmake/CMakeDefinitions.cmake b/cmake/CMakeDefinitions.cmake
@@ -32,6 +32,7 @@ endif()
 set(_comp_flags
   ENABLE_MPI
   ENABLE_HIP
+  ENABLE_OPENMP
   ENABLE_CUDA
   ENABLE_NAN_EXCEPTIONS
   ENABLE_OPENSUBDIV

diff --git a/cmake/Compilers.cmake b/cmake/Compilers.cmake
@@ -1,7 +1,7 @@
 message("-- C++ Compiler ID: ${CMAKE_CXX_COMPILER_ID}")
 
 #-------------------------------------------------------------------------------
-# Optionally suppress compiler warnings
+# Set compiler options
 #-------------------------------------------------------------------------------
 
 option(ENABLE_WARNINGS "show compiler warnings" ON)
@@ -16,12 +16,18 @@ if (ENABLE_HIP)
   set(LANG_STR "HIP")
 endif()
 
-set(CXX_WARNING_FLAGS "")
+set(CXX_COMPILE_FLAGS "")
 if (ENABLE_WARNINGS)
   if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-    list(APPEND CXX_WARNING_FLAGS -fdiagnostics-show-option -Wno-unused-command-line-argument -Wno-c++17-extensions)
+    list(APPEND CXX_COMPILE_FLAGS -fdiagnostics-show-option -Wno-unused-command-line-argument -Wno-c++17-extensions)
     if(CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL 18.0.0)
-      list(APPEND CXX_WARNING_FLAGS -Wno-enum-constexpr-conversion -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments)
+      list(APPEND CXX_COMPILE_FLAGS -Wno-deprecated-declarations -Wno-gnu-zero-variadic-macro-arguments)
+      if(CMAKE_CXX_COMPILER_VERSION LESS 20.0.0)
+        list(APPEND CXX_COMPILE_FLAGS -Wno-enum-constexpr-conversion)
+        # We build some Fortran code from outside sources (like the Helmholtz EOS) that
+        # cause building errors if the compiler is too picky...
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wno-missing-include-dirs")
+      endif()
     endif()
   endif()
 else()
@@ -31,49 +37,62 @@ message("-- Compiler warnings ${ENABLE_WARNINGS}")
 
 if (ENABLE_WARNINGS_AS_ERRORS)
   if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
-    list(APPEND CXX_WARNING_FLAGS /W4 /WX)
+    list(APPEND CXX_COMPILE_FLAGS /W4 /WX)
   else()
-    list(APPEND CXX_WARNING_FLAGS -Wall -Wextra -pedantic -Werror -Wl,--fatal-warnings)
+    list(APPEND CXX_COMPILE_FLAGS -Wall -Wextra -pedantic -Werror -Wl,--fatal-warnings)
   endif()
   message("-- Treating warnings as errors")
 endif()
 
 
 if (NOT ENABLE_UNUSED_VARIABLE_WARNINGS)
-  list(APPEND CXX_WARNING_FLAGS -Wno-unused-variable)
+  list(APPEND CXX_COMPILE_FLAGS -Wno-unused-variable)
 endif()
 message("-- Compiler unused variable warnings ${ENABLE_UNUSED_VARIABLE_WARNINGS}")
 
 
 if (NOT ENABLE_UNUSED_PARAMETER_WARNINGS)
-  list(APPEND CXX_WARNING_FLAGS -Wno-unused-parameter)
+  list(APPEND CXX_COMPILE_FLAGS -Wno-unused-parameter)
 endif()
 message("-- Compiler unused parameter warnings ${ENABLE_UNUSED_PARAMETER_WARNINGS}")
 
 
 if (NOT ENABLE_MISSING_INCLUDE_DIR_WARNINGS)
-  list(APPEND CXX_WARNING_FLAGS -Wno-missing-include-dirs)
+  list(APPEND CXX_COMPILE_FLAGS -Wno-missing-include-dirs)
 endif()
 message("-- Compiler missing include dir warnings ${ENABLE_MISSING_INCLUDE_DIR_WARNINGS}")
 
 set(CUDA_WARNING_FLAGS -Xcudafe=\"--diag_suppress=esa_on_defaulted_function_ignored\")
 
-set_property(GLOBAL PROPERTY SPHERAL_CXX_FLAGS
-  "$<$<COMPILE_LANGUAGE:${LANG_STR}>:${CXX_WARNING_FLAGS}>")
-message("-- Using CXX warning flags ${CXX_WARNING_FLAGS}")
+if (SPHERAL_ENABLE_RDC AND ENABLE_HIP)
+  list(APPEND CXX_COMPILE_FLAGS -fgpu-rdc)
+endif()
+
+set_property(GLOBAL PROPERTY SPHERAL_CXX_FLAGS "${SPHERAL_CXX_FLAGS}"
+  "$<$<COMPILE_LANGUAGE:${LANG_STR}>:${CXX_COMPILE_FLAGS}>")
+message("-- Using CXX compile flags ${CXX_COMPILE_FLAGS}")
 
 # Currently unused
 set_property(GLOBAL PROPERTY SPHERAL_CUDA_FLAGS
   "$<$<COMPILE_LANGUAGE:CUDA>:${CUDA_WARNING_FLAGS}>")
 
-# We build some Fortran code from outside sources (like the Helmholtz EOS) that
-# cause building errors if the compiler is too picky...
-set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Wno-missing-include-dirs")
+#-------------------------------------------------------------------------------
+# Set link options
+#-------------------------------------------------------------------------------
+
+set(CXX_LINK_FLAGS "")
+
+if (SPHERAL_ENABLE_RDC AND ENABLE_HIP)
+  list(APPEND CXX_LINK_FLAGS -fgpu-rdc)
+endif()
+
+set_property(GLOBAL PROPERTY SPHERAL_LINK_FLAGS "${CXX_LINK_FLAGS}")
+message("-- Using link flags ${CXX_LINK_FLAGS}")
 
 #-------------------------------------------------------------------------------
 # PYB11 Target Flags
 #-------------------------------------------------------------------------------
-set(SPHERAL_PYB11_FLAGS ${CXX_WARNING_FLAGS})
+set(SPHERAL_PYB11_FLAGS ${CXX_COMPILE_FLAGS})
 list(APPEND SPHERAL_PYB11_FLAGS
   -O1
   -Wno-unused-local-typedefs 

diff --git a/cmake/SetupSpheral.cmake b/cmake/SetupSpheral.cmake
@@ -66,13 +66,14 @@ elseif(ENABLE_STATIC_CXXONLY)
 endif()
 
 if(ENABLE_MPI)
-  set(SPHERAL_ENABLE_MPI ON)
   set(BLT_MPI_COMPILE_FLAGS -DMPICH_SKIP_MPICXX -ULAM_WANT_MPI2CPP -DOMPI_SKIP_MPICXX)
   list(APPEND SPHERAL_CXX_DEPENDS mpi)
+  set(SPHERAL_ENABLE_MPI ON)
 endif()
 
 if(ENABLE_OPENMP)
   list(APPEND SPHERAL_CXX_DEPENDS openmp)
+  set(SPHERAL_ENABLE_OPENMP ON)
 endif()
 
 if(ENABLE_CUDA)
@@ -137,7 +138,7 @@ set(CMAKE_SKIP_BUILD_RPATH FALSE)
 # (but later on when installing)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
 
-set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}")
+set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib")
 
 # add the automatically determined parts of the RPATH
 # which point to directories outside the build tree to the install RPATH

diff --git a/cmake/SpheralMacros.cmake b/cmake/SpheralMacros.cmake
@@ -59,20 +59,22 @@ macro(spheral_add_test)
     get_property(SPHERAL_BLT_DEPENDS GLOBAL PROPERTY SPHERAL_BLT_DEPENDS)
     get_property(SPHERAL_COMPILE_DEFS GLOBAL PROPERTY SPHERAL_COMPILE_DEFS)
     get_property(SPHERAL_CXX_FLAGS GLOBAL PROPERTY SPHERAL_CXX_FLAGS)
+    get_property(SPHERAL_LINK_FLAGS GLOBAL PROPERTY SPHERAL_LINK_FLAGS)
 
     blt_add_library(
       NAME ${original_test_name}_lib
       SOURCES ${TEST_LIB_SOURCE} ${SPHERAL_ROOT_DIR}/src/spheralCXX.cc
       DEFINES ${SPHERAL_COMPILE_DEFS}
       DEPENDS_ON ${SPHERAL_BLT_DEPENDS} ${original_deps}
-      SHARED False)
+      SHARED FALSE)
 
     target_compile_options(${original_test_name}_lib PUBLIC ${SPHERAL_CXX_FLAGS})
-    target_link_options(${original_test_name}_lib PRIVATE "-Wl,--unresolved-symbols=ignore-in-object-files")
+    target_link_options(${original_test_name}_lib PRIVATE "-Wl,--unresolved-symbols=ignore-in-object-files" ${SPHERAL_LINK_FLAGS})
 
     spheral_add_executable(
       NAME ${original_test_name}
       SOURCES ${original_src}
+      DEFINES ${SPHERAL_COMPILE_DEFS}
       DEPENDS_ON gtest ${CMAKE_THREAD_LIBS_INIT} ${original_test_name}_lib
       TEST On)
 
@@ -81,16 +83,17 @@ macro(spheral_add_test)
       COMMAND ${TEST_DRIVER} ${original_test_name})
 
     target_include_directories(${original_test_name} SYSTEM PRIVATE ${SPHERAL_ROOT_DIR}/tests/cpp/include)
+    target_compile_options(${original_test_name} PUBLIC ${SPHERAL_CXX_FLAGS})
 
     if (${arg_DEBUG_LINKER})
-      target_link_options(${original_test_name} PUBLIC "-Wl,--warn-unresolved-symbols")
+      target_link_options(${original_test_name} PUBLIC "-Wl,--warn-unresolved-symbols" ${SPHERAL_LINK_FLAGS})
     else()
-      target_link_options(${original_test_name} PUBLIC "-Wl,--unresolved-symbols=ignore-all")
+      target_link_options(${original_test_name} PUBLIC "-Wl,--unresolved-symbols=ignore-all" ${SPHERAL_LINK_FLAGS})
     endif()
   endif()
 
   # If we're configuring for unified memory on Cray machines we need to fiddle with the environment
-  if (SPHERAL_UNIFIED_MEMORY AND ENABLE_HIP)
+  if (ENABLE_HIP)
     set_tests_properties(${original_test_name} PROPERTIES ENVIRONMENT HSA_XNACK=1)
   endif()
 

diff --git a/cmake/SpheralOptions.cmake b/cmake/SpheralOptions.cmake
@@ -40,6 +40,16 @@ endif()
 option(SPHERAL_ENABLE_STATIC "Building static C++ libraries" ${DEFAULT_STATIC})
 cmake_dependent_option(SPHERAL_ENABLE_SHARED "Building shared C++ libraries" ON "NOT SPHERAL_ENABLE_STATIC" OFF)
 
+#-------------------------------------------------------------------------------
+# Experimental/unused options
+#-------------------------------------------------------------------------------
+
+cmake_dependent_option(SPHERAL_ENABLE_RDC "Build using the RDC flag. Currently broken (DO NOT USE)" OFF SPHERAL_GPU_ENABLED OFF)
+
+if (SPHERAL_ENABLE_RDC AND SPHERAL_ENABLE_SHARED)
+  message(FATAL_ERROR "Must use SPHERAL_ENABLE_STATIC if enabling SPHERAL_ENABLE_RDC (once it is working)")
+endif()
+
 #-------------------------------------------------------------------------------
 # Should we build sphinx documentation
 #-------------------------------------------------------------------------------

diff --git a/cmake/spheral/SpheralAddLibs.cmake b/cmake/spheral/SpheralAddLibs.cmake
@@ -8,6 +8,7 @@
 # SPHERAL_CXX_DEPENDS    : REQUIRED : List of compiler dependencies
 # SPHERAL_COMPILE_DEFS   : REQUIRED : List of compiler definitions
 # SPHERAL_CXX_FLAGS      : REQUIRED : List of C++ compiler options
+# SPHERAL_LINK_FLAGS     : REQUIRED : List of link options
 # <package_name>_headers : OPTIONAL : List of necessary headers to include
 # <package_name>_sources : OPTIONAL : List of necessary source files to include
 # SPHERAL_SUBMOD_DEPENDS : REQUIRED : List of submodule dependencies
@@ -32,6 +33,8 @@ function(spheral_add_obj_library package_name obj_list_name)
   get_property(SPHERAL_COMPILE_DEFS GLOBAL PROPERTY SPHERAL_COMPILE_DEFS)
   # Assumes global variable SPHERAL_CXX_FLAGS exists and is filled with C++ compiler options
   get_property(SPHERAL_CXX_FLAGS GLOBAL PROPERTY SPHERAL_CXX_FLAGS)
+  # Assumes global variable SPHERAL_LINK_FLAGS exists and is filled with linker options
+  get_property(SPHERAL_LINK_FLAGS GLOBAL PROPERTY SPHERAL_LINK_FLAGS)
   # For including files in submodules, currently unused
   get_property(SPHERAL_SUBMOD_INCLUDES GLOBAL PROPERTY SPHERAL_SUBMOD_INCLUDES)
 
@@ -42,6 +45,7 @@ function(spheral_add_obj_library package_name obj_list_name)
       DEFINES     ${SPHERAL_COMPILE_DEFS}
       DEPENDS_ON  ${SPHERAL_CXX_DEPENDS} ${SPHERAL_BLT_DEPENDS} 
       SHARED      TRUE)
+    target_link_options(Spheral_${package_name} PUBLIC ${SPHERAL_LINK_FLAGS})
   else()
     blt_add_library(NAME Spheral_${package_name}
       HEADERS     ${${package_name}_headers}
@@ -82,6 +86,7 @@ endfunction()
 # SPHERAL_CXX_DEPENDS    : REQUIRED : List of compiler dependencies
 # SPHERAL_COMPILE_DEFS   : REQUIRED : List of compiler definitions
 # SPHERAL_CXX_FLAGS      : REQUIRED : List of C++ compiler options
+# SPHERAL_LINK_FLAGS     : REQUIRED : List of link options
 # <package_name>_headers : OPTIONAL : List of necessary headers to include
 # <package_name>_sources : OPTIONAL : List of necessary source files to include
 # SPHERAL_SUBMOD_DEPENDS : REQUIRED : List of submodule dependencies
@@ -104,6 +109,8 @@ function(spheral_add_cxx_library package_name _cxx_obj_list)
   get_property(SPHERAL_COMPILE_DEFS GLOBAL PROPERTY SPHERAL_COMPILE_DEFS)
   # Assumes global variable SPHERAL_CXX_FLAGS exists and is filled with C++ compiler options
   get_property(SPHERAL_CXX_FLAGS GLOBAL PROPERTY SPHERAL_CXX_FLAGS)
+  # Assumes global variable SPHERAL_LINK_FLAGS exists and is filled with linker options
+  get_property(SPHERAL_LINK_FLAGS GLOBAL PROPERTY SPHERAL_LINK_FLAGS)
   # For including files in submodules, currently unused
   get_property(SPHERAL_SUBMOD_INCLUDES GLOBAL PROPERTY SPHERAL_SUBMOD_INCLUDES)
   # Convert package name to lower-case for export target name
@@ -124,11 +131,12 @@ function(spheral_add_cxx_library package_name _cxx_obj_list)
 
     # Add compile options
     target_compile_options(Spheral_${package_name} PRIVATE ${SPHERAL_CXX_FLAGS})
+    target_link_options(Spheral_${package_name} PRIVATE ${SPHERAL_LINK_FLAGS})
   endif()
 
   target_include_directories(Spheral_${package_name} SYSTEM PRIVATE ${SPHERAL_SUBMOD_INCLUDES})
 
-  if(ENABLE_CUDA)
+  if(ENABLE_CUDA AND SPHERAL_ENABLE_RDC)
     set_target_properties(Spheral_${package_name} PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
   endif()
 
@@ -139,9 +147,6 @@ function(spheral_add_cxx_library package_name _cxx_obj_list)
 
   # Export Spheral target
   install(EXPORT ${export_target_name} DESTINATION lib/cmake)
-
-  # Set the r-path of the C++ lib such that it is independent of the build dir when installed
-  set_target_properties(Spheral_${package_name} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
 endfunction()
 
 #----------------------------------------------------------------------------------------
@@ -285,8 +290,7 @@ function(spheral_add_pybind11_library package_name module_list_name)
           DESTINATION ${SPHERAL_SITE_PACKAGES_PATH}/Spheral)
 
   set_property(GLOBAL APPEND PROPERTY ${module_list_name} ${package_name})
-
-  # Set the r-path of the C++ lib such that it is independent of the build dir when installed
-  set_target_properties(${MODULE_NAME} PROPERTIES INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+  get_property(SPHERAL_LINK_FLAGS GLOBAL PROPERTY SPHERAL_LINK_FLAGS)
+  target_link_options(Spheral${package_name} PUBLIC ${SPHERAL_LINK_FLAGS})
 
 endfunction()
diff --git a/scripts/spack/configs/toss_4_x86_64_ib_cray/compilers.yaml b/scripts/spack/configs/toss_4_x86_64_ib_cray/compilers.yaml
@@ -1,11 +1,11 @@
 compilers::
 - compiler:
-      spec: rocmcc@=6.2.0
+      spec: rocmcc@=7.1.0
       paths:
-        cc:  /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/bin/amdclang
-        cxx: /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/bin/amdclang++
-        f77: /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/bin/amdflang
-        fc:  /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/bin/amdflang
+        cc:  /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/bin/amdclang
+        cxx: /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/bin/amdclang++
+        f77: /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/bin/amdflang
+        fc:  /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/bin/amdflang
       flags: {}
       operating_system: rhel8
       target: x86_64

diff --git a/scripts/spack/configs/toss_4_x86_64_ib_cray/packages.yaml b/scripts/spack/configs/toss_4_x86_64_ib_cray/packages.yaml
@@ -29,45 +29,45 @@ packages:
   cray-mpich:
     buildable: false
     externals:
-    - spec: cray-mpich@8.1.31%rocmcc@6.2.0
-      prefix: /usr/tce/packages/cray-mpich/cray-mpich-8.1.31-rocmcc-6.2.0-magic
+    - spec: cray-mpich@9.0.1%rocmcc@7.1.0
+      prefix: /usr/tce/packages/cray-mpich/cray-mpich-9.0.1-rocmcc-7.1.0-magic
 
   hip:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: hip@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/
+    - spec: hip@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/
   llvm-amdgpu:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: llvm-amdgpu@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocmcc-6.2.0-magic/llvm
+    - spec: llvm-amdgpu@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocmcc-7.1.0-magic/llvm
   hsa-rocr-dev:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: hsa-rocr-dev@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocm-6.2.0-magic/
+    - spec: hsa-rocr-dev@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocm-7.1.0-magic/
   rocminfo:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: rocminfo@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocm-6.2.0-magic/
+    - spec: rocminfo@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocm-7.1.0-magic/
   rocm-device-libs:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: rocm-device-libs@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocm-6.2.0-magic/
+    - spec: rocm-device-libs@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocm-7.1.0-magic/
   rocprim:
-    version: [6.2.0]
+    version: [7.1.0]
     buildable: false
     externals:
-    - spec: rocprim@6.2.0%rocmcc@6.2.0
-      prefix: /usr/tce/packages/rocmcc/rocm-6.2.0-magic/
+    - spec: rocprim@7.1.0%rocmcc@7.1.0
+      prefix: /usr/tce/packages/rocmcc/rocm-7.1.0-magic/
 
 # ------ SYSTEM LIBS -------
   py-decorator:

diff --git a/scripts/spack/environments/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack/environments/toss_4_x86_64_ib_cray/spack.yaml
@@ -7,6 +7,7 @@ spack:
   view: false
   concretizer:
     unify: false
+    reuse: false
   develop:
     spheral:
       path: ../../../../