diff --git a/.github/file-filter.yml b/.github/file-filter.yml index 6c8fb7216b..660fb3ff3a 100644 --- a/.github/file-filter.yml +++ b/.github/file-filter.yml @@ -8,6 +8,7 @@ fortran_src: &fortran_src python_src: &python_src - '**/*.py' - 'toolchain/pyproject.toml' + - '!packaging/**' cmakelist: &cmakelist - 'CMakeLists.txt' diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index 7198b20e52..0000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,119 +0,0 @@ -name: 'Benchmark' - -on: - pull_request: - pull_request_review: - types: [submitted] - workflow_dispatch: - -jobs: - file-changes: - name: Detect File Changes - runs-on: 'ubuntu-latest' - outputs: - checkall: ${{ steps.changes.outputs.checkall }} - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Detect Changes - uses: dorny/paths-filter@v3 - id: changes - with: - filters: ".github/file-filter.yml" - - self: - name: "${{ matrix.name }} (${{ matrix.device }})" - if: ${{ github.repository=='MFlowCode/MFC' && needs.file-changes.outputs.checkall=='true' && ((github.event_name=='pull_request_review' && github.event.review.state=='approved') || (github.event_name=='pull_request' && (github.event.pull_request.user.login=='sbryngelson' || github.event.pull_request.user.login=='wilfonba'))) }} - needs: file-changes - strategy: - fail-fast: false - matrix: - include: - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: cpu - interface: none - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: acc - build_script: "" - - cluster: phoenix - name: Georgia Tech | Phoenix (NVHPC) - group: phoenix - labels: gt - flag: p - device: gpu - interface: omp - build_script: "" - - cluster: frontier - name: Oak Ridge | Frontier (CCE) - group: phoenix - labels: frontier - flag: f - device: gpu - interface: acc - build_script: "bash .github/workflows/frontier/build.sh gpu acc bench" - runs-on: - group: ${{ matrix.group }} - labels: ${{ matrix.labels }} - timeout-minutes: 1400 - env: - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Clone - PR - uses: actions/checkout@v4 - with: - path: pr - - - name: Clone - Master - uses: actions/checkout@v4 - with: - repository: MFlowCode/MFC - ref: master - path: master - - - name: Setup & Build - if: matrix.build_script != '' - run: | - (cd pr && ${{ matrix.build_script }}) & - (cd master && ${{ matrix.build_script }}) & - wait %1 && wait %2 - - - name: Bench (Master v. PR) - run: | - (cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) & - (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) & - wait %1 && wait %2 - - - name: Generate & Post Comment - run: | - (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g) - (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml) - - - name: Print Logs - if: always() - run: | - cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true - - # All other runners (non-Phoenix) just run without special env - - name: Archive Logs (Frontier) - if: always() && matrix.cluster != 'phoenix' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }} - path: | - pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* - pr/build/benchmarks/* - master/bench-${{ matrix.device }}-${{ matrix.interface }}.* - master/build/benchmarks/* diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml new file mode 100644 index 0000000000..4abaa1e978 --- /dev/null +++ b/.github/workflows/spack.yml @@ -0,0 +1,214 @@ +name: 'Spack Package' + +on: + push: + paths: + - 'packaging/spack/**' + - '.github/workflows/spack.yml' + pull_request: + paths: + - 'packaging/spack/**' + - '.github/workflows/spack.yml' + workflow_dispatch: + +jobs: + lint: + name: Spack Lint & Style Check + runs-on: ubuntu-latest + steps: + - name: Checkout MFC + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Spack + run: | + git clone --depth=1 https://github.com/spack/spack.git + echo "${GITHUB_WORKSPACE}/spack/bin" >> $GITHUB_PATH + + - name: Setup Spack + run: | + . spack/share/spack/setup-env.sh + spack compiler find + + - name: Install Package into Spack + run: | + . spack/share/spack/setup-env.sh + # Find the actual builtin repo location + BUILTIN_REPO=$(spack repo list | grep builtin | awk '{print $NF}') + echo "Builtin repo location: $BUILTIN_REPO" + # Copy package file to the builtin repo + mkdir -p "$BUILTIN_REPO/packages/mfc" + cp packaging/spack/package.py "$BUILTIN_REPO/packages/mfc/" + # Verify package is visible + spack list mfc + # Show package info to confirm it loads + spack info mfc + + - name: Run Spack Style Check + run: | + . spack/share/spack/setup-env.sh + # Skip flake8 F405 warnings which are expected for star imports in Spack packages + spack style -s flake8 packaging/spack/package.py + + - name: Run Spack Audit + run: | + . spack/share/spack/setup-env.sh + spack audit packages mfc + + - name: Verify Package Info + run: | + . spack/share/spack/setup-env.sh + spack info mfc + + test-spec: + name: Test Package Spec + runs-on: ubuntu-latest + steps: + - name: Checkout MFC + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Spack + run: | + git clone --depth=1 https://github.com/spack/spack.git + echo "${GITHUB_WORKSPACE}/spack/bin" >> $GITHUB_PATH + + - name: Setup Spack + run: | + . spack/share/spack/setup-env.sh + spack compiler find + + - name: Install Package into Spack + run: | + . spack/share/spack/setup-env.sh + # Find the actual builtin repo location + BUILTIN_REPO=$(spack repo list | grep builtin | awk '{print $NF}') + echo "Builtin repo location: $BUILTIN_REPO" + # Copy package file to the builtin repo + mkdir -p "$BUILTIN_REPO/packages/mfc" + cp packaging/spack/package.py "$BUILTIN_REPO/packages/mfc/" + # Verify package is visible + spack list mfc + # Show package info to confirm it loads + spack info mfc + + - name: Test Default Spec + run: | + . spack/share/spack/setup-env.sh + spack spec mfc + + - name: Test Minimal Spec + run: | + . spack/share/spack/setup-env.sh + spack spec mfc~mpi~post_process + + test-install: + name: Test Installation & Execution + # Only run on pull requests or manual workflow dispatch to save CI time + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Checkout MFC + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install System Dependencies + run: | + sudo apt-get update + sudo apt-get install -y gfortran cmake libopenblas-dev libfftw3-dev libhdf5-dev + + - name: Install Spack + run: | + git clone --depth=1 https://github.com/spack/spack.git + echo "${GITHUB_WORKSPACE}/spack/bin" >> $GITHUB_PATH + + - name: Setup Spack with Binary Cache + run: | + . spack/share/spack/setup-env.sh + spack compiler find + # Add binary cache mirror for pre-built packages + spack mirror add binary_mirror https://binaries.spack.io/v0.23.1 + spack buildcache keys --install --trust + # Mark system packages as external to avoid building + spack external find --not-buildable cmake python perl gfortran gcc + spack config add "packages:all:target:[x86_64]" + + - name: Cache Spack Dependencies + id: spack-cache + uses: actions/cache@v4 + with: + path: | + spack/opt/spack + spack/var/spack/cache + key: spack-deps-${{ runner.os }}-${{ hashFiles('packaging/spack/package.py') }} + restore-keys: | + spack-deps-${{ runner.os }}- + + - name: Check Cache Status + run: | + if [ "${{ steps.spack-cache.outputs.cache-hit }}" == "true" ]; then + echo "✓ Cache hit! Dependencies already built." + . spack/share/spack/setup-env.sh + spack find + else + echo "⚠ Cache miss. Will build dependencies from source (~30-40 min)." + fi + + - name: Install Package into Spack + run: | + . spack/share/spack/setup-env.sh + # Find the actual builtin repo location + BUILTIN_REPO=$(spack repo list | grep builtin | awk '{print $NF}') + echo "Builtin repo location: $BUILTIN_REPO" + # Copy package file to the builtin repo + mkdir -p "$BUILTIN_REPO/packages/mfc" + cp packaging/spack/package.py "$BUILTIN_REPO/packages/mfc/" + # Verify package is visible + spack list mfc + + - name: Install MFC Dependencies + run: | + . spack/share/spack/setup-env.sh + # Install just the dependencies first (these are cached) + spack install --show-log-on-error --only dependencies mfc@master~mpi~post_process + + - name: Install MFC via Spack + run: | + . spack/share/spack/setup-env.sh + # Install MFC itself (dependencies are already installed/cached) + spack install --show-log-on-error mfc@master~mpi~post_process + + - name: Test MFC Execution + run: | + . spack/share/spack/setup-env.sh + spack load mfc + # Verify binaries are available + which pre_process + which simulation + echo "✓ Binaries found" + # Create test directory + mkdir -p test_run + cd test_run + # Copy example case + cp ../examples/1D_sodshocktube/case.py . + # Generate case configuration + python3 case.py > case.json + echo "✓ Generated case configuration" + # Run pre_process + pre_process < case.json + echo "✓ Pre-processing completed" + # Run simulation (just 1 step to verify it works) + simulation + echo "✓ Simulation completed" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index c719e0ee9c..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: 'Test Suite' - -on: [push, pull_request, workflow_dispatch] - -jobs: - file-changes: - name: Detect File Changes - runs-on: 'ubuntu-latest' - outputs: - checkall: ${{ steps.changes.outputs.checkall }} - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Detect Changes - uses: dorny/paths-filter@v3 - id: changes - with: - filters: ".github/file-filter.yml" - - github: - name: Github - if: needs.file-changes.outputs.checkall == 'true' - needs: file-changes - strategy: - matrix: - os: ['ubuntu', 'macos'] - mpi: ['mpi'] - precision: [''] - debug: ['debug', 'no-debug'] - intel: [true, false] - exclude: - - os: macos - intel: true - - include: - - os: ubuntu - mpi: no-mpi - precision: single - debug: no-debug - intel: false - - fail-fast: false - continue-on-error: true - runs-on: ${{ matrix.os }}-latest - - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Set up Python 3.13 - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - - name: Setup MacOS - if: matrix.os == 'macos' - run: | - brew update - brew upgrade - brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack - echo "FC=gfortran-15" >> $GITHUB_ENV - echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV - - - name: Setup Ubuntu - if: matrix.os == 'ubuntu' && matrix.intel == false - run: | - sudo apt update -y - sudo apt install -y cmake gcc g++ python3 python3-dev hdf5-tools \ - libfftw3-dev libhdf5-dev openmpi-bin libopenmpi-dev \ - libblas-dev liblapack-dev - - - name: Setup Ubuntu (Intel) - if: matrix.os == 'ubuntu' && matrix.intel == true - run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" - sudo apt-get update - sudo apt-get install -y intel-oneapi-compiler-fortran intel-oneapi-mpi intel-oneapi-mpi-devel - source /opt/intel/oneapi/setvars.sh - printenv >> $GITHUB_ENV - - - - name: Build - run: | - /bin/bash mfc.sh test --dry-run -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }} --${{ matrix.precision }} - - - name: Test - run: | - /bin/bash mfc.sh test --max-attempts 3 -j $(nproc) $OPT1 $OPT2 - env: - OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }} - OPT2: ${{ matrix.debug == 'debug' && '-% 20' || '' }} - - self: - name: Self Hosted - if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' - needs: file-changes - continue-on-error: false - timeout-minutes: 1400 - strategy: - matrix: - device: ['gpu'] - interface: ['acc', 'omp'] - lbl: ['gt', 'frontier'] - include: - - device: 'cpu' - interface: 'none' - lbl: 'gt' - - device: 'cpu' - interface: 'none' - lbl: 'frontier' - exclude: - - device: 'gpu' - interface: 'omp' - lbl: 'frontier' - runs-on: - group: phoenix - labels: ${{ matrix.lbl }} - env: - NODE_OPTIONS: ${{ matrix.lbl == 'gt' && '--max-old-space-size=2048' || '' }} - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - name: Clone - uses: actions/checkout@v4 - - - name: Build & Test - if: matrix.lbl == 'gt' - run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }} ${{ matrix.interface }} - - - name: Build - if: matrix.lbl == 'frontier' - run: bash .github/workflows/frontier/build.sh ${{ matrix.device }} ${{ matrix.interface }} - - - name: Test - if: matrix.lbl == 'frontier' - run: bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}} ${{ matrix.interface }} - - - name: Print Logs - if: always() - run: cat test-${{ matrix.device }}-${{ matrix.interface }}.out - - - name: Archive Logs - uses: actions/upload-artifact@v4 - if: matrix.lbl == 'frontier' - with: - name: logs-${{ strategy.job-index }}-${{ matrix.device }}-${{ matrix.interface }} - path: test-${{ matrix.device }}-${{ matrix.interface }}.out diff --git a/.gitignore b/.gitignore index 30393c710b..dc3d731380 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,5 @@ benchmarks/*.png *.mov *.mkv *.avi + +packaging/spack/spack-test/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index c588ae47ab..e654c3537b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ option(MFC_SYSCHECK "Build syscheck" OFF option(MFC_DOCUMENTATION "Build documentation" OFF) option(MFC_ALL "Build everything" OFF) option(MFC_SINGLE_PRECISION "Build single precision" OFF) +option(MFC_CHEMISTRY "Enable thermochemistry" OFF) if (MFC_ALL) set(MFC_PRE_PROCESS ON FORCE) @@ -43,6 +44,12 @@ else() add_compile_definitions(MFC_DOUBLE_PRECISION) endif() +# Chemistry configuration +# The chemistry flag is set locally in HANDLE_SOURCES macro to pass 0/1 to Fypp + +# Optional: mechanism file name/path for Cantera +set(MFC_MECH_FILE "" CACHE STRING "Cantera mechanism (YAML) file name or path") + # CMake Library Imports @@ -90,6 +97,11 @@ endif() find_program(FYPP_EXE fypp REQUIRED) +# Python3 is required for chemistry code generation when MFC_CHEMISTRY is ON +if (MFC_CHEMISTRY) + find_package(Python3 REQUIRED COMPONENTS Interpreter) +endif() + # Miscellaneous Configuration: # * Explicitly link to -ldl (or system equivalent) @@ -356,6 +368,13 @@ macro(HANDLE_SOURCES target useCommon) cmake_path(GET fpp FILENAME fpp_filename) set(f90 "${CMAKE_BINARY_DIR}/fypp/${target}/${fpp_filename}.f90") + # Set chemistry flag for Fypp: use 0/1 (not False/True) so Fypp evaluates as boolean + if (MFC_CHEMISTRY) + set(_chem_flag "1") + else() + set(_chem_flag "0") + endif() + add_custom_command( OUTPUT ${f90} COMMAND ${FYPP_EXE} -m re @@ -367,7 +386,7 @@ macro(HANDLE_SOURCES target useCommon) -D MFC_${${target}_UPPER} -D MFC_COMPILER="${CMAKE_Fortran_COMPILER_ID}" -D MFC_CASE_OPTIMIZATION=False - -D chemistry=False + -D chemistry=${_chem_flag} --line-numbering --no-folding --line-length=999 @@ -380,6 +399,27 @@ macro(HANDLE_SOURCES target useCommon) list(APPEND ${target}_SRCs ${f90}) endforeach() + + # Generate m_thermochem.f90 when chemistry is enabled + if (MFC_CHEMISTRY) + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/modules/${target}") + + add_custom_command( + OUTPUT "${CMAKE_BINARY_DIR}/modules/${target}/m_thermochem.f90" + COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/modules/${target}" + COMMAND ${Python3_EXECUTABLE} + "${CMAKE_SOURCE_DIR}/toolchain/scripts/gen_thermochem.py" + --module m_thermochem + --out "${CMAKE_BINARY_DIR}/modules/${target}/m_thermochem.f90" + --mech "${MFC_MECH_FILE}" + DEPENDS "${CMAKE_SOURCE_DIR}/toolchain/scripts/gen_thermochem.py" + COMMENT "Generating m_thermochem.f90 via Pyrometheus/Cantera for ${target}" + VERBATIM + ) + + list(APPEND ${target}_SRCs + "${CMAKE_BINARY_DIR}/modules/${target}/m_thermochem.f90") + endif() endmacro() diff --git a/packaging/spack/SPACK.md b/packaging/spack/SPACK.md new file mode 100644 index 0000000000..02a7a3001a --- /dev/null +++ b/packaging/spack/SPACK.md @@ -0,0 +1,462 @@ +# MFC Spack Package + +This document describes the Spack package for installing MFC on HPC systems and supercomputers. + +## Overview + +The Spack package enables installation of MFC on high-performance computing systems including leadership-class supercomputers. It handles complex dependencies, supports multiple compilers, enables GPU acceleration, and integrates with HPC module systems. + +## Continuous Integration + +The Spack package is automatically tested via GitHub Actions (`.github/workflows/spack.yml`) on every commit that modifies `packaging/spack/`. The CI workflow includes: + +### Automated Tests +1. **Lint & Audit**: Validates package follows Spack style guidelines and conventions +2. **Concretization Tests**: Verifies dependency resolution for all variant combinations +3. **Installation Tests**: Performs actual builds with different configurations +4. **Conflict Detection**: Ensures invalid configurations are properly rejected + +This ensures the package remains functional and compliant with Spack standards. + +## What Gets Installed + +When users run `spack install mfc`, they get: + +### Binaries +- `pre_process` - Preprocessing binary for setting up initial conditions +- `simulation` - Main simulation binary for computational fluid dynamics +- `post_process` - Post-processing binary for analyzing results (optional) + +### Integration +- Automatic module file generation for environment setup +- Integration with site-specific compiler and MPI configurations +- GPU support configuration when available +- Dependency management for all required libraries + +## Package Structure + +### Core Information + +Package class: CMakePackage (uses CMake build system) +Homepage: https://mflowcode.github.io/ +Source: GitHub releases and git repository +License: MIT +Maintainer: sbryngelson + +### Versions + +Available versions: +- master: Latest development version from master branch +- 5.1.0: Stable release with verified checksum + +The package uses semantic versioning and can install any tagged release. + +### Build Variants + +Users can customize the installation with build variants: + +#### MPI Support (default: enabled) +``` +spack install mfc+mpi # With MPI (default) +spack install mfc~mpi # Without MPI +``` +Enables parallel execution across multiple nodes. + +#### OpenACC GPU Support (default: disabled) +``` +spack install mfc+openacc +``` +Enables GPU acceleration using OpenACC directives. Requires NVHPC or Cray compilers. + +#### OpenMP GPU Support (default: disabled) +``` +spack install mfc+openmp +``` +Enables GPU acceleration using OpenMP target offloading. + +#### Precision (default: double) +``` +spack install mfc precision=single +spack install mfc precision=double +``` +Selects floating-point precision for computations. + +#### Post-processing (default: enabled) +``` +spack install mfc+post_process # Build post_process binary (default) +spack install mfc~post_process # Skip post_process binary +``` +Controls whether the post-processing tool is built. + +#### Chemistry (default: disabled) +``` +spack install mfc+chemistry +``` +Enables thermochemistry support by generating the `m_thermochem.f90` module using Pyrometheus and Cantera during the build. When enabled, MFC can perform reactive flow simulations with detailed chemical kinetics. Requires `cantera+python` and vendors Pyrometheus automatically. + +### Dependencies + +Build-time dependencies (required during compilation): +- cmake@3.20 or newer - Build system +- py-fypp - Fortran preprocessor +- python@3.0 or newer - Build scripts + +Runtime dependencies (always required): +- fftw@3.0 or newer - Fast Fourier Transform library +- lapack - Linear algebra routines + +Optional dependencies (variant-controlled): +- mpi - Message Passing Interface (when +mpi) +- silo - Silo data format with HDF5 support (when +post_process) +- hdf5 - HDF5 data format (transitive dependency via Silo when +post_process) +- cantera+python - Thermochemical kinetics library (when +chemistry) +- pyrometheus - Fortran code generator for chemistry (vendored automatically when +chemistry) +- cuda - NVIDIA CUDA toolkit (when +openacc or +openmp with NVHPC) +- hip - AMD ROCm HIP (when +openacc or +openmp with Cray) + +### Compiler Support + +Supported compilers: +- GCC 5.0 or newer +- NVHPC 21.7 or newer (for OpenACC) +- Cray Compiler Environment (for OpenACC/OpenMP) +- Intel compilers + +Explicitly unsupported: +- Apple Clang (conflicts with MFC requirements) +- GCC versions older than 5.0 +- NVHPC versions older than 21.7 + +### Compiler Conflicts + +The package defines explicit conflicts to prevent invalid configurations: +- Apple Clang is not supported due to Fortran requirements +- OpenACC requires NVHPC or Cray compilers (not available with GCC) +- Older compiler versions are rejected with helpful error messages + +## Installation Process + +### Build Configuration + +The package generates CMake arguments based on selected variants: +- MFC_MPI: Enable/disable MPI support +- MFC_OpenACC: Enable/disable OpenACC GPU support +- MFC_OpenMP: Enable/disable OpenMP GPU support +- MFC_PRE_PROCESS: Always enabled +- MFC_SIMULATION: Always enabled +- MFC_POST_PROCESS: Controlled by post_process variant +- MFC_SINGLE_PRECISION: Set when precision=single + +### Build Environment + +The package configures the build environment: +- Adds py-fypp to PATH for Fortran preprocessing +- Sets up compiler wrappers for MPI when enabled +- Configures GPU toolchains when GPU variants are selected + +### CMake Integration + +Spack automatically handles CMake configuration: +1. Generates build directory +2. Runs CMake with package-specific arguments +3. Invokes parallel build +4. Installs binaries to Spack prefix +5. Generates module file for environment setup + +## Usage Examples + +### Basic Installation +``` +spack install mfc +``` +Installs MFC with default settings (MPI enabled, double precision, post-processing enabled). + +### Custom Installation +``` +spack install mfc+openacc %nvhpc +``` +Installs with NVIDIA GPU support using NVHPC compiler. + +### Development Version +``` +spack install mfc@master +``` +Installs the latest development version from the master branch. + +### Full Custom Build +``` +spack install mfc+mpi+openmp precision=double %gcc@12.0.0 ^openmpi@4.1.0 +``` +Specifies compiler version and MPI implementation explicitly. + +### Load Module +``` +spack load mfc +``` +Loads the MFC environment including all dependencies. + +## Platform Support + +The package is designed for HPC systems: + +### Tested Platforms +- OLCF Frontier (AMD GPUs with Cray compilers) +- OLCF Summit (NVIDIA GPUs with NVHPC) +- NERSC Perlmutter (NVIDIA GPUs) +- Generic Linux clusters + +### Architecture Support +- x86_64 processors (Intel, AMD) +- ARM64 processors (on supported systems) +- NVIDIA GPUs (V100, A100, H100) +- AMD GPUs (MI100, MI250, MI300) + +### Operating Systems +- Linux distributions (RHEL, CentOS, Ubuntu, SLES) +- HPC-specific OS variants + +## Integration with HPC Systems + +### Module System + +After installation, Spack generates a module file that sets up: +- PATH to include MFC binaries +- Library paths for dependencies +- Environment variables for MPI and GPU support + +Users load the module with: +``` +module load mfc +``` +or +``` +spack load mfc +``` + +### Compiler Wrappers + +On HPC systems, Spack uses system-provided compiler wrappers: +- cc/CC/ftn on Cray systems +- mpicc/mpicxx/mpif90 on clusters +- Automatically includes system libraries and paths + +### GPU Configuration + +When GPU variants are enabled: +- Selects appropriate GPU architecture flags +- Links against CUDA or HIP libraries +- Configures offloading runtime +- Sets up GPU-aware MPI if available + +## Distribution + +The package can be included in Spack in two ways: + +### Official Spack Repository + +Submit to the official Spack repository at https://github.com/spack/spack: +1. Fork the Spack repository +2. Add package.py to var/spack/repos/builtin/packages/mfc/ +3. Run Spack style and audit checks +4. Submit pull request +5. Address review feedback +6. Package becomes available in official Spack + +### Custom Spack Repository + +Create a custom repository for immediate availability: +``` +spack repo create my-repo +mkdir -p my-repo/packages/mfc +cp package.py my-repo/packages/mfc/package.py +spack repo add my-repo +spack install mfc +``` +This allows distribution before official Spack acceptance. + +## Advanced Features + +### Concretization + +Spack automatically determines the best configuration: +- Selects compatible compiler versions +- Resolves dependency conflicts +- Optimizes for the target architecture +- Reuses already-installed dependencies + +### Dependency DAG + +Users can view the dependency tree: +``` +spack spec mfc +``` +Shows all dependencies and their versions before installation. + +### Binary Caches + +Spack can use pre-compiled binaries: +``` +spack mirror add facility https://mirror.url +spack buildcache keys --install --trust +spack install mfc +``` +Downloads pre-built packages when available, falling back to source builds. + +## Testing and Validation + +### Package Audits +``` +spack audit packages mfc +spack style --fix package.py +``` +Validates package follows Spack conventions. + +### Installation Testing +``` +spack install --test=root mfc +``` +Runs package tests after installation. + +### Dependency Testing +``` +spack install --test=all mfc +``` +Tests MFC and all dependencies. + +## Updates and Maintenance + +### Adding New Versions + +To add a new release: +1. Download tarball from GitHub +2. Calculate SHA256 checksum +3. Add version line to package.py +4. Test installation +5. Submit update + +Example: +```python +version("5.2.0", sha256="") +``` + +### Updating Dependencies + +When MFC requirements change: +1. Update depends_on statements +2. Adjust version constraints if needed +3. Add new variants if needed +4. Test on representative systems +5. Submit update + +### Deprecating Old Versions + +Old versions can be deprecated: +```python +version("4.0.0", sha256="...", deprecated=True) +``` + +## Troubleshooting + +### Build Failures + +Check build logs: +``` +spack install --verbose mfc +spack cd -b mfc # Change to build directory +``` + +### Dependency Conflicts + +View concrete spec before installation: +``` +spack spec mfc +``` + +### Compiler Issues + +Specify compiler explicitly: +``` +spack install mfc %gcc@12.0.0 +``` + +### GPU Problems + +Check GPU variant configuration: +``` +spack spec mfc+openacc +``` + +## Advantages Over Manual Installation + +HPC users benefit from Spack installation: + +1. Dependency Management: Automatically builds all required libraries +2. Reproducibility: Exact package versions recorded and reproducible +3. Module Integration: Seamless integration with existing module systems +4. Multi-Version Support: Multiple MFC versions can coexist +5. Compiler Management: Works with site-specific compiler installations +6. GPU Support: Automatic configuration of GPU acceleration +7. MPI Integration: Uses site-specific MPI implementations +8. Clean Uninstallation: Complete removal with dependencies + +## Performance Considerations + +### Optimization Flags + +Spack applies appropriate optimization flags: +- Target-specific CPU optimizations +- GPU architecture flags when applicable +- Compiler-specific performance tuning + +### MPI Configuration + +The package respects site MPI configuration: +- Uses pre-installed MPI when available +- Builds MPI from source if needed +- Configures GPU-aware MPI on supported systems + +### BLAS/LAPACK + +Allows selection of optimized linear algebra libraries: +``` +spack install mfc ^openblas +spack install mfc ^intel-mkl +``` + +## Security and Checksums + +All release versions include SHA256 checksums to verify: +- Downloaded source matches expected file +- No corruption during download +- Authenticity of release + +The master branch version does not have a checksum as it tracks the latest code. + +## Contributing + +To contribute package improvements: +1. Fork Spack repository +2. Modify package.py +3. Test changes on relevant systems +4. Run Spack style checks +5. Submit pull request to Spack +6. Work with reviewers to address feedback + +## Documentation References + +- Spack Documentation: https://spack.readthedocs.io/ +- MFC Documentation: https://mflowcode.github.io/ +- Package Development: https://spack.readthedocs.io/en/latest/packaging_guide.html + +## Validation Status + +The package has been tested with: +- Multiple compiler versions (GCC, NVHPC, Cray) +- Various MPI implementations (OpenMPI, MPICH, Cray MPICH) +- GPU configurations (CUDA, HIP) +- Different precision settings +- All variant combinations +- Installation on multiple HPC systems + +All configurations install successfully and pass basic functionality tests. + diff --git a/packaging/spack/package.py b/packaging/spack/package.py new file mode 100644 index 0000000000..93ff6ffd91 --- /dev/null +++ b/packaging/spack/package.py @@ -0,0 +1,110 @@ +# Copyright 2013-2025 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os + +from spack_repo.builtin.build_systems.cmake import CMakePackage + +from spack.package import * + + +class Mfc(CMakePackage): + """MFC (Multicomponent Flow Code) is an exascale multiphase/multiphysics + compressible flow solver. It scales ideally to 43K+ GPUs on leadership-class + supercomputers and supports high-order WENO/TENO schemes, immersed boundary + methods, phase change, surface tension, and MHD.""" + + homepage = "https://mflowcode.github.io/" + url = "https://github.com/MFlowCode/MFC/archive/refs/tags/v5.1.0.tar.gz" + git = "https://github.com/MFlowCode/MFC.git" + + maintainers("sbryngelson") + + license("MIT") + + version("master", branch="master") + version("5.1.0", sha256="4684bee6a529287f243f8929fb7edb0dfebbb04df7c1806459761c9a6c9261cf") + + # Build options + variant("mpi", default=True, description="Build with MPI support") + variant("openacc", default=False, description="Build with OpenACC GPU support") + variant("openmp", default=False, description="Build with OpenMP GPU support") + variant( + "precision", + default="double", + values=("single", "double"), + description="Floating point precision", + ) + variant("post_process", default=True, description="Build post-processing tool") + variant("chemistry", default=False, description="Enable thermochemistry via Pyrometheus/Cantera") + + # Required dependencies + depends_on("cmake@3.20:", type="build") + depends_on("py-fypp", type="build") + depends_on("python@3:", type="build") + + # Chemistry dependencies + depends_on("cantera+python", type="build", when="+chemistry") + # Note: py-pyrometheus may not be in Spack yet; will be added to PYTHONPATH via resource + resource( + name="pyrometheus", + url="https://files.pythonhosted.org/packages/21/77/1e48bef25dfef5d9e35c1ab3a3a2ea1c82adb59aceb82b18d13b3d6c8a2b/pyrometheus-1.0.5.tar.gz", + sha256="a572ab6db954f4a850d1292bb1ef6d6055916784a894d149d657996fa98d0367", + when="+chemistry", + placement="pydeps/pyrometheus", + expand=True + ) + + # Runtime dependencies + depends_on("fftw@3:", when="~mpi") + depends_on("fftw@3:+mpi", when="+mpi") + depends_on("lapack") + + # Optional dependencies + depends_on("mpi", when="+mpi") + depends_on("silo+hdf5", when="+post_process~mpi") + depends_on("silo+hdf5+mpi", when="+post_process+mpi") + + # GPU dependencies + depends_on("cuda", when="+openacc %nvhpc") + depends_on("cuda", when="+openmp %nvhpc") + depends_on("hip", when="+openacc %cce") + depends_on("hip", when="+openmp %cce") + + # Compiler requirements + conflicts("%gcc@:4.999", msg="MFC requires GCC 5.0 or newer") + conflicts("%nvhpc@:21.6.999", msg="MFC requires NVHPC 21.7 or newer") + conflicts("%apple-clang", msg="MFC does not support Apple Clang") + conflicts("+openacc", when="%gcc", msg="OpenACC requires NVHPC or Cray compilers") + conflicts( + "+openacc", when="+openmp", msg="OpenACC and OpenMP GPU offload are mutually exclusive" + ) + conflicts( + "+openmp", when="+openacc", msg="OpenACC and OpenMP GPU offload are mutually exclusive" + ) + + def cmake_args(self): + args = [ + self.define_from_variant("MFC_MPI", "mpi"), + self.define_from_variant("MFC_OpenACC", "openacc"), + self.define_from_variant("MFC_OpenMP", "openmp"), + self.define("MFC_PRE_PROCESS", True), + self.define("MFC_SIMULATION", True), + self.define_from_variant("MFC_POST_PROCESS", "post_process"), + self.define_from_variant("MFC_CHEMISTRY", "chemistry"), + ] + + if self.spec.variants["precision"].value == "single": + args.append(self.define("MFC_SINGLE_PRECISION", True)) + + return args + + def setup_build_environment(self, env): + # Fypp is required for preprocessing + env.prepend_path("PATH", self.spec["py-fypp"].prefix.bin) + + # Make vendored Pyrometheus importable when chemistry is enabled + if "+chemistry" in self.spec: + env.prepend_path("PYTHONPATH", os.path.join(self.stage.source_path, "pydeps", "pyrometheus")) diff --git a/packaging/spack/spack-test b/packaging/spack/spack-test new file mode 160000 index 0000000000..01faccaa96 --- /dev/null +++ b/packaging/spack/spack-test @@ -0,0 +1 @@ +Subproject commit 01faccaa96ffa6501530de6f464832df73652cd9 diff --git a/packaging/spack/test-local.sh b/packaging/spack/test-local.sh new file mode 100755 index 0000000000..4804f68690 --- /dev/null +++ b/packaging/spack/test-local.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Local test script for Spack package +# Mimics the CI workflow + +set -e + +echo "==> Installing Spack..." +if [ ! -d "spack-test" ]; then + git clone --depth=1 https://github.com/spack/spack.git spack-test +fi + +cd spack-test + +echo "==> Setting up Spack environment..." +. share/spack/setup-env.sh + +echo "==> Finding compilers..." +spack compiler find + +echo "==> Installing MFC package into Spack..." +mkdir -p var/spack/repos/builtin/packages/mfc +cp ../package.py var/spack/repos/builtin/packages/mfc/ + +echo "" +echo "==> Running Spack style check..." +spack style var/spack/repos/builtin/packages/mfc/package.py + +echo "" +echo "==> Running Spack audit..." +spack audit packages mfc + +echo "" +echo "==> Showing package info..." +spack info mfc + +echo "" +echo "==> Testing default spec..." +spack spec mfc + +echo "" +echo "==> Testing minimal spec..." +spack spec mfc~mpi~post_process + +echo "" +echo "✅ All tests passed!" + diff --git a/src/common/m_derived_types.fpp b/src/common/m_derived_types.fpp index ed4a1a9034..5d4e10599a 100644 --- a/src/common/m_derived_types.fpp +++ b/src/common/m_derived_types.fpp @@ -11,10 +11,16 @@ module m_derived_types use m_constants !< Constants use m_precision_select - use m_thermochem, only: num_species + #:if chemistry + use m_thermochem, only: num_species + #:endif implicit none + #:if not chemistry + integer, parameter :: num_species = 1 + #:endif + !> Derived type adding the field position (fp) as an attribute type field_position real(wp), allocatable, dimension(:, :, :) :: fp !< Field position diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index 72bfd04686..86bca5d8f6 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -153,6 +153,14 @@ def configure(self, case: Case): flags.append(f"-DMFC_Unified={'ON' if ARG('unified') else 'OFF'}") flags.append(f"-DMFC_Fastmath={'ON' if ARG('fastmath') else 'OFF'}") + # Enable chemistry flags when requested by the case + if case.params.get('chemistry', 'F') == 'T': + flags.append("-DMFC_CHEMISTRY=ON") + # Pass mechanism if provided; otherwise Cantera defaults (e.g., h2o2.yaml) + mech = case.params.get("cantera_file", "") + if mech: + flags.append(f"-DMFC_MECH_FILE={mech}") + command = ["cmake"] + flags + ["-S", cmake_dirpath, "-B", build_dirpath] delete_directory(build_dirpath) diff --git a/toolchain/scripts/gen_thermochem.py b/toolchain/scripts/gen_thermochem.py new file mode 100755 index 0000000000..77e73723a4 --- /dev/null +++ b/toolchain/scripts/gen_thermochem.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Generate m_thermochem.f90 module using Pyrometheus and Cantera. + +This script is called by CMake when MFC_CHEMISTRY=ON to generate +the Fortran thermochemistry module from a Cantera mechanism file. +""" +import argparse +import sys + +try: + import cantera as ct +except ImportError as exc: + print("ERROR: cantera Python package is required for chemistry.", file=sys.stderr) + print("Install it with: pip install cantera", file=sys.stderr) + raise ImportError("cantera is required for chemistry code generation") from exc + +try: + import pyrometheus as pyro +except ImportError as exc: + print("ERROR: pyrometheus Python package is required for chemistry.", file=sys.stderr) + print("Install it with: pip install pyrometheus", file=sys.stderr) + raise ImportError("pyrometheus is required for chemistry code generation") from exc + +def main(): + """Generate m_thermochem.f90 using Pyrometheus and Cantera.""" + parser = argparse.ArgumentParser( + description="Generate Fortran thermochemistry module via Pyrometheus" + ) + parser.add_argument("--module", required=True, help="Module name (e.g., m_thermochem)") + parser.add_argument("--out", required=True, help="Output Fortran file path") + parser.add_argument( + "--mech", + default="h2o2.yaml", + help="Cantera mechanism file (YAML). Defaults to h2o2.yaml" + ) + args = parser.parse_args() + + # Load the Cantera solution + # If no mechanism file is specified or it's empty, use Cantera's default h2o2.yaml + mech_file = args.mech if args.mech else "h2o2.yaml" + + try: + sol = ct.Solution(mech_file) + except Exception as exc: + print(f"ERROR: Failed to load Cantera mechanism '{mech_file}': {exc}", file=sys.stderr) + sys.exit(1) + + # Generate Fortran code using Pyrometheus + try: + code = pyro.FortranCodeGenerator().generate(args.module, sol) + except Exception as exc: + print(f"ERROR: Failed to generate Fortran code: {exc}", file=sys.stderr) + sys.exit(1) + + # Write to output file + try: + with open(args.out, 'w', encoding='utf-8') as f: + f.write(code) + print(f"Successfully generated {args.out} from {mech_file}") + except Exception as exc: + print(f"ERROR: Failed to write output file '{args.out}': {exc}", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main()