merge

Author: wilfonba

.github/workflows/frontier/build.sh

@@ -13,6 +13,6 @@ if [ "$2" == "bench" ]; then
         ./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts
     done
 else
-    ./mfc.sh test -a --dry-run --rdma-mpi --generate -j 8 $build_opts
+    ./mfc.sh test -a --dry-run --rdma-mpi -j 8 $build_opts
 fi
 

.github/workflows/frontier/submit.sh

@@ -33,7 +33,7 @@ sbatch <<EOT
 #SBATCH -A CFD154                  # charge account
 #SBATCH -N 1                       # Number of nodes required
 $sbatch_device_opts
-#SBATCH -t 04:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -t 05:59:00                # Duration of the job (Ex: 15 mins)
 #SBATCH -o$job_slug.out            # Combined output and error messages file
 #SBATCH -p extended                # Extended partition for shorter queues
 #SBATCH -W                         # Do not exit until the submitted job terminates.

.github/workflows/frontier/test.sh

@@ -6,5 +6,5 @@ ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
 if [ "$job_device" = "gpu" ]; then
     ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus -- -c frontier
 else
-    ./mfc.sh test -a --rdma-mpi --max-attempts 3 -j 32 -- -c frontier
+    ./mfc.sh test -a --max-attempts 3 -j 32 -- -c frontier
 fi

.github/workflows/test.yml

@@ -48,6 +48,11 @@ jobs:
       - name: Clone
         uses: actions/checkout@v4
 
+      - name: Set up Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+
       - name: Setup MacOS
         if:   matrix.os == 'macos'
         run:  |

.typos.toml

@@ -19,6 +19,7 @@ strang = "strang"
 Strang = "Strang"
 TKE = "TKE"
 HSA = "HSA"
+infp = "infp"
 
 [files]
 extend-exclude = ["docs/documentation/references*", "tests/", "toolchain/cce_simulation_workgroup_256.sh"]

.vscode/settings.json

@@ -49,7 +49,7 @@
     // Enable ONLY fortls language server
     "fortran.enableLanguageServer": true,
     "fortran.languageServer": "fortls",
-    "fortran.fortls.disabled": false,
+    "fortran.fortls.disabled": true,
     "fortran.fortls.path": "fortls",
 
     // Try to disable any built-in language features

CMakeLists.txt

@@ -328,6 +328,13 @@ macro(HANDLE_SOURCES target useCommon)
                                                "${CMAKE_BINARY_DIR}/modules/${target}/*.fpp")
     if (${useCommon})
         file(GLOB common_FPPs CONFIGURE_DEPENDS "${common_DIR}/*.fpp")
+
+        # If we're building post_process, exclude m_compute_levelset.fpp
+        if("${target}" STREQUAL "post_process")
+            list(FILTER common_FPPs EXCLUDE REGEX ".*/m_compute_levelset\.fpp$")
+            list(FILTER common_FPPs EXCLUDE REGEX ".*/m_ib_patches\.fpp$")
+        endif()
+
         list(APPEND ${target}_FPPs ${common_FPPs})
     endif()
 
@@ -443,7 +450,7 @@ function(MFC_SETUP_TARGET)
 
         if (ARGS_SILO)
             find_package(SILO REQUIRED)
-            target_link_libraries(${a_target} PRIVATE SILO::SILO)
+            target_link_libraries(${a_target} PRIVATE SILO::SILO stdc++)
         endif()
 
         if (ARGS_HDF5)

README.md

@@ -28,7 +28,7 @@
 **Welcome!**
 MFC simulates compressible multi-phase flows, [among other things](#what-else-can-this-thing-do). 
 It uses metaprogramming to stay short and portable (~20K lines).
-MFC conducted the largest known, open CFD simulation at <a href="https://arxiv.org/abs/2505.07392" target="_blank">101 trillion grid points</a> (as of July 2025).
+MFC conducted the largest known, open CFD simulation at <a href="https://arxiv.org/abs/2505.07392" target="_blank">200 trillion grid points</a>, and 1 quadrillion degrees of freedom (as of September 2025), and is a 2025 Gordon Bell Prize finalist.
 
 <p align="center">
 <a href="https://doi.org/10.48550/arXiv.2503.07953" target="_blank">
@@ -187,7 +187,7 @@ They are organized below.
 
 * GPU compatible on NVIDIA ([P/V/A/H]100, GH200, etc.) and AMD (MI[1/2/3]00+) GPU and APU hardware
 * Ideal weak scaling to 100% of the largest GPU and superchip supercomputers
- 	* \>36K AMD APUs (MI300A) on [LLNL El Capitan](https://hpc.llnl.gov/hardware/compute-platforms/el-capitan)
+ 	* \>43K AMD APUs (MI300A) on [LLNL El Capitan](https://hpc.llnl.gov/hardware/compute-platforms/el-capitan)
    	* \>3K AMD APUs (MI300A) on [LLNL Tuolumne](https://hpc.llnl.gov/hardware/compute-platforms/tuolumne)
 	* \>33K AMD GPUs (MI250X) on [OLCF Frontier](https://www.olcf.ornl.gov/frontier/) 
 	* \>10K NVIDIA GPUs (V100) on [OLCF Summit](https://www.olcf.ornl.gov/summit/) 
@@ -199,7 +199,7 @@ They are organized below.
 
 * [Fypp](https://fypp.readthedocs.io/en/stable/fypp.html) metaprogramming for code readability, performance, and portability
 * Continuous Integration (CI)
-	* > 500 Regression tests with each PR.
+	* \>500 Regression tests with each PR.
  		* Performed with GNU (GCC), Intel (oneAPI), Cray (CCE), and NVIDIA (NVHPC) compilers on NVIDIA and AMD GPUs.
 		* Line-level test coverage reports via [Codecov](https://app.codecov.io/gh/MFlowCode/MFC) and `gcov`
 	* Benchmarking to avoid performance regressions and identify speed-ups

docs/documentation/case.md

@@ -482,9 +482,9 @@ It is recommended to set `weno_eps` to $10^{-6}$ for WENO-JS, and to $10^{-40}$
 
 - `mp_weno` activates monotonicity preservation in the WENO reconstruction (MPWENO) such that the values of reconstructed variables do not reside outside the range spanned by WENO stencil ([Balsara and Shu, 2000](references.md); [Suresh and Huynh, 1997](references.md)).
 
-- `muscl_order` specifies the order of the MUSCL scheme that is used for spatial reconstruction of variables by an integer of 1, or 2, that corresponds to the 1st, and 2nd order respectively. When using `muscl_order = 2`, `muscl_lim` must be defined. 
+- `muscl_order` specifies the order of the MUSCL scheme that is used for spatial reconstruction of variables by an integer of 1, or 2, that corresponds to the 1st, and 2nd order respectively. When using `muscl_order = 2`, `muscl_lim` must be defined.
 
-- `muscl_lim` specifies the slope limiter that is used in 2nd order MUSCL Reconstruction by an integer from 1 through 5. 
+- `muscl_lim` specifies the slope limiter that is used in 2nd order MUSCL Reconstruction by an integer from 1 through 5.
 `muscl_lim = 1`, `2`, `3`, `4`, and `5` correspond to minmod, monotonized central, Van Albada, Van Leer, and SUPERBEE, respectively.
 
 - `int_comp` activates interface compression using THINC used in MUSCL Reconstruction, with control parameters (`ic_eps`, and `ic_beta`).
@@ -599,6 +599,24 @@ To restart the simulation from $k$-th time step, see [Restarting Cases](running.
 | `output_partial_domain` | Logical | Output part of the domain |
 | `[x,y,z]_output%beg` | Real    | Beginning of the output domain in the [x,y,z]-direction |
 | `[x,y,z]_output%end` | Real    | End of the output domain in the [x,y,z]-direction |
+| `lag_txt_wrt`        | Logical | Write Lagrangian bubble data to `.dat` files |
+| `lag_header`         | Logical | Write header to Lagrangian bubble `.dat` files |
+| `lag_db_wrt`         | Logical | Write Lagrangian bubble data to silo/hdf5 database files |
+| `lag_id_wrt`         | Logical | Add the global bubble idea to the database file |
+| `lag_pos_wrt`        | Logical | Add the bubble position to the database file |
+| `lag_pos_prev_wrt`   | Logical | Add the previous bubble position to the database file |
+| `lag_vel_wrt`        | Logical | Add the bubble translational velocity to the database file |
+| `lag_rad_wrt`        | Logical | Add the bubble radius to the database file |
+| `lag_rvel_wrt`       | Logical | Add the bubble radial velocity to the database file |
+| `lag_r0_wrt`         | Logical | Add the bubble initial radius to the database file |
+| `lag_rmax_wrt`       | Logical | Add the bubble maximum radius to the database file |
+| `lag_rmin_wrt`       | Logical | Add the bubble minimum radius to the database file |
+| `lag_dphidt_wrt`     | Logical | Add the bubble subgrid velocity potential to the database file |
+| `lag_pres_wrt`       | Logical | Add the bubble pressure to the database file |
+| `lag_mv_wrt`         | Logical | Add the bubble vapor mass to the database file |
+| `lag_mg_wrt`         | Logical | Add the bubble gas mass to the database file |
+| `lag_betaT_wrt`      | Logical | Add the bubble heat flux model coefficient to the database file |
+| `lag_betaC_wrt`      | Logical | Add the bubble mass flux model coefficient to the database file |
 
 The table lists formatted database output parameters. The parameters define variables that are outputted from simulation and file types and formats of data as well as options for post-processing.
 
@@ -628,7 +646,7 @@ If `file_per_process` is true, then pre_process, simulation, and post_process mu
 
 - `output_partial_domain` activates the output of part of the domain specified by `[x,y,z]_output%beg` and `[x,y,z]_output%end`.
 This is useful for large domains where only a portion of the domain is of interest.
-It is not supported when `precision = 1` and `format = 1`. 
+It is not supported when `precision = 1` and `format = 1`.
 It also cannot be enabled with `flux_wrt`, `heat_ratio_wrt`, `pres_inf_wrt`, `c_wrt`, `omega_wrt`, `ib`, `schlieren_wrt`, `qm_wrt`, or 'liutex_wrt'.
 
 ### 8. Acoustic Source {#acoustic-source}

docs/documentation/gpuParallelization.md

@@ -31,7 +31,7 @@ Note: Ordering is not guaranteed or stable, so use key-value pairing when using
 
 - Data on the GPU has a reference counter
 - When data is referred to being allocated, it means that GPU memory is allocated if it is not already present in GPU memory. If a variable is already present, the reference counter is just incremented.
-- When data is referred to being dellocated, it means that the reference counter is decremented. If the reference counter is zero, then the data is actually deallocated from GPU memory
+- When data is referred to being deallocated, it means that the reference counter is decremented. If the reference counter is zero, then the data is actually deallocated from GPU memory
 - When data is referred to being attached, it means that the device pointer attaches to target if it not already attached. If pointer is already attached, then the attachment counter is just incremented
 - When data is referred to being detached, it means that the attachment counter is decremented. If attachment counter is zero, then actually detached