CMake: AMReX_FASTMATH (#4545)

## Summary

Add new general compiler flag for fast-math optimizations.
https://en.wikipedia.org/wiki/Floating-point_arithmetic#%22Fast_math%22_optimization

**This PR does not yet change any defaults (see details below).**

Close #1769

## Proposed Breaking Change

By default, AMReX turned fast-math OFF for all backends (serial,
CPU/OpenMP, SYCL GPU, HIP GPU) besides:
- CUDA GPUs (both GNUmake & CMake)
- a single, legacy [cray compiler in
GNUmake](https://github.com/AMReX-Codes/amrex/blob/25.07/Tools/GNUMake/comps/cray.mak)

I propose to amend this PR to:
- remove the `AMReX_CUDA_FASTMATH` flag (default: ON)
- alternatively: fade it out and set its default value to
`AMReX_FASTMATH`
- turn `AMReX_FASTMATH` to `OFF` by default for all backends.

Contrary to turning it `ON` by default, users of most backends will not
be surprised by a sudden change in numerical results.

### Breaking:

- Users that support fast-math, which might be many will see a sudden
drop of performance on CUDA GPUs.

### Recommended migration:

- If you already benchmarked correctness on CUDA GPUs, I recommend to
intentionally set in your projects `-DAMReX_FASTMATH=ON` for all
backends, to get the performance benefits on CPUs, Intel and AMD GPUs as
well.


## Additional background

As HPC team, we should strive to make our numerics as robust as possible
under fast-math optimizations. Without those, your compiler will barely
auto-vectorize, will not replace simple optimizations like `x / 2.` with
`x * 0.5` and many other optimizations that rely on agressive
re-ordering.

Fast-math floating point math is an important aspect to consider and
support for significant performance improvements on all modern
architectures.

## Checklist

The proposed changes:
- [x] fix a bug or incorrect behavior in AMReX
- [x] add new capabilities to AMReX
- [x] changes answers in the test suite to more than roundoff level
- [x] are likely to significantly affect the results of downstream AMReX
users
- [x] include documentation in the code and/or rst files, if appropriate

Author: ax3l

Docs/sphinx_documentation/source/BuildingAMReX.rst

@@ -443,6 +443,8 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_BUILD_SHARED_LIBS      |  Build as shared C++ library                    | NO (unless xSDK)        | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_FASTMATH               |  Enable fast-math optimizations                 | NO (CPU), YES (CUDA)    | YES, NO               |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_FORTRAN                |  Enable Fortran language                        | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_PRECISION              |  Set the precision of reals                     | DOUBLE                  | DOUBLE, SINGLE        |

Tools/CMake/AMReXCUDAOptions.cmake

@@ -26,7 +26,7 @@ if(DEFINED ENV{AMREX_CUDA_ARCH})
 endif()
 set(AMReX_CUDA_ARCH ${AMReX_CUDA_ARCH_DEFAULT} CACHE STRING "CUDA architecture (Use 'Auto' for automatic detection)")
 
-option(AMReX_CUDA_FASTMATH "Enable CUDA fastmath" ON)
+option(AMReX_CUDA_FASTMATH "Enable CUDA fastmath" ON)  # Note: inconsistent with AMReX_FASTMATH defaults
 cuda_print_option( AMReX_CUDA_FASTMATH )
 
 set(AMReX_CUDA_MAXREGCOUNT "255" CACHE STRING

Tools/CMake/AMReXConfig.cmake.in

@@ -125,6 +125,7 @@ set(AMReX_HIP                       @AMReX_HIP@)
 set(AMReX_GPU_BACKEND               @AMReX_GPU_BACKEND@)
 set(AMReX_GPU_RDC                   @AMReX_GPU_RDC@)
 set(AMReX_PRECISION                 @AMReX_PRECISION@)
+set(AMReX_FASTMATH                  @AMReX_FASTMATH@)
 set(AMReX_FORTRAN                   @AMReX_FORTRAN@)
 
 # Actual components selection

Tools/CMake/AMReXOptions.cmake

@@ -253,6 +253,18 @@ cmake_dependent_option( AMReX_GPU_RDC "Enable Relocatable Device Code" ${_GPU_RD
 unset(_GPU_RDC_default)
 print_option(AMReX_GPU_RDC)
 
+#
+# Fast Math    ================================================================
+#
+set(_FASTMATH_default OFF)
+if(AMReX_GPU_BACKEND STREQUAL CUDA)  # note: historic settings
+# if(NOT AMReX_GPU_BACKEND STREQUAL NONE)  # note: this would be more consistent for GPUs
+    set(_FASTMATH_default ON)
+endif()
+option(AMReX_FASTMATH  "Enable fast-math optimizations" ${_FASTMATH_default})
+print_option(AMReX_FASTMATH)
+unset(_FASTMATH_default)
+
 #
 # Parallel backends    ========================================================
 #

Tools/CMake/AMReXParallelBackends.cmake

@@ -205,6 +205,11 @@ if (AMReX_SYCL)
    include(AMReXSYCL)
    foreach(D IN LISTS AMReX_SPACEDIM)
       target_link_libraries(amrex_${D}d PUBLIC SYCL)
+
+       # fast math
+       if(AMReX_FASTMATH)
+           target_compile_options(amrex_${D}d PUBLIC -ffast-math)
+       endif()
    endforeach()
 endif ()
 
@@ -350,6 +355,11 @@ if (AMReX_HIP)
    foreach(D IN LISTS AMReX_SPACEDIM)
        target_compile_options(amrex_${D}d PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-m64>)
 
+       # fast math
+       if(AMReX_FASTMATH)
+           target_compile_options(amrex_${D}d PUBLIC -ffast-math)
+       endif()
+
        # ROCm 4.5: use unsafe floating point atomics, otherwise atomicAdd is much slower
        # 
        target_compile_options(amrex_${D}d PUBLIC $<$<COMPILE_LANGUAGE:CXX>:-munsafe-fp-atomics>)

Tools/CMake/AMReX_Config.cmake

@@ -72,7 +72,6 @@ function (configure_amrex AMREX_TARGET)
    )
 
    unset(_condition)
-   unset(_cxx_msvc)
 
    #
    # Setup OpenMP
@@ -146,11 +145,33 @@ function (configure_amrex AMREX_TARGET)
       endif()
    endif()
 
+   # fast math
+   if (AMReX_FASTMATH)
+       # GPU specific backends set in AMReXParallelBackends.cmake
+       if (AMReX_GPU_BACKEND STREQUAL NONE)
+           # See https://cmake.org/cmake/help/v4.1/variable/CMAKE_LANG_COMPILER_ID.html#variable:CMAKE_%3CLANG%3E_COMPILER_ID
+           target_compile_options(${AMREX_TARGET} PUBLIC
+               $<$<CXX_COMPILER_ID:AppleClang,Clang,CrayClang,GNU,IBMClang,IntelLLVM,XLClang>:-ffast-math>
+               $<${_cxx_msvc}:"/fp:fast">  # MSVC
+           )
+           if (CMAKE_Fortran_COMPILER_LOADED)
+               target_compile_options(${AMREX_TARGET} PUBLIC
+                   $<$<Fortran_COMPILER_ID:AppleClang,Clang,CrayClang,GNU,IBMClang,IntelLLVM,XLClang>:-ffast-math>
+               )
+           endif ()
+       endif()
+   endif()
+
    #
    # Setup third-party profilers
    #
    set_amrex_profilers(${AMREX_TARGET})
 
+   #
+   # clean up helpers
+   #
+   unset(_cxx_msvc)
+
 endfunction ()
 
 #

Tools/CMake/AMReX_Config_ND.H.in

@@ -35,6 +35,7 @@
 #cmakedefine AMREX_USE_FLATTEN_FOR
 #cmakedefine AMREX_BOUND_CHECK
 #cmakedefine AMREX_EXPORT_DYNAMIC
+#cmakedefine AMREX_FASTMATH
 #cmakedefine BL_FORT_USE_UNDERSCORE
 #cmakedefine BL_FORT_USE_LOWERCASE
 #cmakedefine BL_FORT_USE_UPPERCASE