diff --git a/.clang-tidy b/.clang-tidy
index 892b1b8a..84742de4 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -1,46 +1,46 @@
-Checks: >
- '-*,
- bugprone-*,
- -bugprone-branch-clone,
- -bugprone-easily-swappable-parameters,
- -bugprone-exception-escape,
- -bugprone-implicit-widening-of-multiplication-result,
- clang-analyzer-*,
- -clang-analyzer-optin.mpi.MPI-Checker,
- clang-diagnostic-*,
- cppcoreguidelines-*,
- -cppcoreguidelines-avoid-c-arrays,
- -cppcoreguidelines-avoid-goto,
- -cppcoreguidelines-avoid-magic-numbers,
- -cppcoreguidelines-avoid-non-const-global-variables,
- -cppcoreguidelines-init-variables,
- -cppcoreguidelines-interfaces-global-init,
- -cppcoreguidelines-macro-usage,
- -cppcoreguidelines-no-malloc,
- -cppcoreguidelines-non-private-member-variables-in-classes,
- -cppcoreguidelines-owning-memory,
- -cppcoreguidelines-pro-*,
- modernize-*,
- -modernize-avoid-c-arrays,
- -modernize-macro-to-enum,
- -modernize-return-braced-init-list,
- -modernize-use-trailing-return-type,
- -modernize-use-using,
- performance-*,
- readability-*,
- -readability-braces-around-statements,
- -readability-container-data-pointer,
- -readability-else-after-return,
- -readability-function-cognitive-complexity,
- -readability-function-size,
- -readability-identifier-length,
- -readability-implicit-bool-conversion,
- -readability-isolate-declaration,
- -readability-magic-numbers,
- -readability-named-parameter,
- -readability-simplify-boolean-expr,
- mpi-*
- '
-
-# Files not ending with nolint.H will be filtered in.
-HeaderFilterRegex: '([^n].....|[^o]....|[^l]...|[^i]..|[^n].|[^t])\.H$'
+Checks: >
+ '-*,
+ bugprone-*,
+ -bugprone-branch-clone,
+ -bugprone-easily-swappable-parameters,
+ -bugprone-exception-escape,
+ -bugprone-implicit-widening-of-multiplication-result,
+ clang-analyzer-*,
+ -clang-analyzer-optin.mpi.MPI-Checker,
+ clang-diagnostic-*,
+ cppcoreguidelines-*,
+ -cppcoreguidelines-avoid-c-arrays,
+ -cppcoreguidelines-avoid-goto,
+ -cppcoreguidelines-avoid-magic-numbers,
+ -cppcoreguidelines-avoid-non-const-global-variables,
+ -cppcoreguidelines-init-variables,
+ -cppcoreguidelines-interfaces-global-init,
+ -cppcoreguidelines-macro-usage,
+ -cppcoreguidelines-no-malloc,
+ -cppcoreguidelines-non-private-member-variables-in-classes,
+ -cppcoreguidelines-owning-memory,
+ -cppcoreguidelines-pro-*,
+ modernize-*,
+ -modernize-avoid-c-arrays,
+ -modernize-macro-to-enum,
+ -modernize-return-braced-init-list,
+ -modernize-use-trailing-return-type,
+ -modernize-use-using,
+ performance-*,
+ readability-*,
+ -readability-braces-around-statements,
+ -readability-container-data-pointer,
+ -readability-else-after-return,
+ -readability-function-cognitive-complexity,
+ -readability-function-size,
+ -readability-identifier-length,
+ -readability-implicit-bool-conversion,
+ -readability-isolate-declaration,
+ -readability-magic-numbers,
+ -readability-named-parameter,
+ -readability-simplify-boolean-expr,
+ mpi-*
+ '
+
+# Files not ending with nolint.H will be filtered in.
+HeaderFilterRegex: '([^n].....|[^o]....|[^l]...|[^i]..|[^n].|[^t])\.H$'
diff --git a/.editorconfig b/.editorconfig
index 446ca9f2..c9b6a028 100644
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,49 +1,49 @@
-# http://EditorConfig.org
-#
-# precedence of rules is bottom to top
-
-# this is the top-most EditorConfig file
-root = true
-
-
-[*.{c,h,cpp,hpp,H,py}]
-# 4 space indentation
-indent_style = space
-indent_size = 4
-
-# setting it to true would result in too many white changes to amrex
-trim_trailing_whitespace = true
-
-# unix-style newlines
-end_of_line = lf
-
-# newline ending in files
-insert_final_newline = true
-
-
-[*.md]
-# two end of line whitespaces are newlines without a paragraph
-trim_trailing_whitespace = false
-
-
-[*.rst]
-# Enforce UTF-8 encoding
-charset = utf-8
-
-# Unix-style newlines
-end_of_line = lf
-
-# Newline ending in files
-insert_final_newline = true
-
-# 3 space indentation
-indent_style = space
-indent_size = 3
-
-# Clean up trailing whitespace
-trim_trailing_whitespace = true
-
-[Makefile]
-# TABs are part of its syntax
-indent_style = tab
-indent_size = unset
+# http://EditorConfig.org
+#
+# precedence of rules is bottom to top
+
+# this is the top-most EditorConfig file
+root = true
+
+
+[*.{c,h,cpp,hpp,H,py}]
+# 4 space indentation
+indent_style = space
+indent_size = 4
+
+# setting it to true would result in too many white changes to amrex
+trim_trailing_whitespace = true
+
+# unix-style newlines
+end_of_line = lf
+
+# newline ending in files
+insert_final_newline = true
+
+
+[*.md]
+# two end of line whitespaces are newlines without a paragraph
+trim_trailing_whitespace = false
+
+
+[*.rst]
+# Enforce UTF-8 encoding
+charset = utf-8
+
+# Unix-style newlines
+end_of_line = lf
+
+# Newline ending in files
+insert_final_newline = true
+
+# 3 space indentation
+indent_style = space
+indent_size = 3
+
+# Clean up trailing whitespace
+trim_trailing_whitespace = true
+
+[Makefile]
+# TABs are part of its syntax
+indent_style = tab
+indent_size = unset
diff --git a/.gitattributes b/.gitattributes
index 87122e07..6771b5c1 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,4 +1,6 @@
-*.ipynb linguist-vendored
-*.tex linguist-documentation
-
-*.H linguist-language=C++
+*.ipynb linguist-vendored
+*.tex linguist-documentation
+
+*.H linguist-language=C++
+# Force use of LF
+* text=auto eol=lf
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 866c2cfe..3ae97540 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,18 +1,18 @@
-nohup.out
-*.[oa]
-*.ex
-*~
-chk?????
-plt?????
-d/
-f/
-o/
-chk?????.old*
-plt?????.old*
-Tutorials_profling/*/*
-log*
-bl_prof*
-case_results*/
-tmp_build_dir/
+nohup.out
+*.[oa]
+*.ex
+*~
+chk?????
+plt?????
+d/
+f/
+o/
+chk?????.old*
+plt?????.old*
+Tutorials_profling/*/*
+log*
+bl_prof*
+case_results*/
+tmp_build_dir/
.codegpt
\ No newline at end of file
diff --git a/JOSS_paper/paper.bib b/JOSS_paper/paper.bib
index 612cff62..b22c1b41 100644
--- a/JOSS_paper/paper.bib
+++ b/JOSS_paper/paper.bib
@@ -1,87 +1,87 @@
-@ARTICLE{almgren1998conservative,
- title={{A conservative adaptive projection method for the variable density incompressible Navier--Stokes equations}},
- author={Almgren, Ann S and Bell, John B and Colella, Phillip and Howell, Louis H and Welcome, Michael L},
- journal={Journal of Computational Physics},
- volume={142},
- number={1},
- pages={1--46},
- year={1998},
- publisher={Elsevier},
- DOI = {10.1006/jcph.1998.5890}
-}
-
-@ARTICLE{zeng2022parallel,
- title={{A parallel cell-centered adaptive level set framework for efficient simulation of two-phase flows with subcycling and non-subcycling}},
- author={Zeng, Yadong and Xuan, Anqing and Blaschke, Johannes and Shen, Lian},
- journal={Journal of Computational Physics},
- volume={448},
- pages={110740},
- year={2022},
- publisher={Elsevier},
- DOI = {10.1016/j.jcp.2021.110740}
-}
-
-@ARTICLE{zeng2023consistent,
- title={{A consistent adaptive level set framework for incompressible two-phase flows with high density ratios and high Reynolds numbers}},
- author={Zeng, Yadong and Liu, Han and Gao, Qiang and Almgren, Ann and Bhalla, Amneet Pal Singh and Shen, Lian},
- journal={Journal of Computational Physics},
- volume={478},
- pages={111971},
- year={2023},
- publisher={Elsevier},
- DOI = {10.1016/j.jcp.2023.111971}
-}
-
-@ARTICLE{li2024open,
- title={{An open-source, adaptive solver for particle-resolved simulations with both subcycling and non-subcycling methods}},
- author={Li, Xuzhu and Li, Chun and Li, Xiaokai and Li, Wenzhuo and Tang, Mingze and Zeng, Yadong and Zhu, Zhengping},
- journal={Physics of Fluids},
- volume={36},
- number={11},
- year={2024},
- publisher={AIP Publishing},
- DOI = {10.1063/5.0236509}
-}
-
-@ARTICLE{zhang2019amrex,
- title={{AMReX: a framework for block-structured adaptive mesh refinement}},
- author={Zhang, Weiqun and Almgren, Ann and Beckner, Vince and Bell, John and Blaschke, Johannes and Chan, Cy and Day, Marcus and Friesen, Brian and Gott, Kevin and Graves, Daniel and others},
- journal={Journal of Open Source Software},
- volume={4},
- number={37},
- year={2019},
- publisher = {The Open Journal},
- DOI = {10.21105/joss.01370}
-}
-
-@article{gong2023cp3d,
- title={CP3d: A comprehensive Euler-Lagrange solver for direct numerical simulation of particle-laden flows},
- author={Gong, Zheng and Wu, Zi and An, Chenge and Zhang, Bangwen and Fu, Xudong},
- journal={Computer Physics Communications},
- volume={286},
- pages={108666},
- year={2023},
- publisher={Elsevier}
-}
-
-@article{costa2018fft,
- title={A FFT-based finite-difference solver for massively-parallel direct numerical simulations of turbulent flows},
- author={Costa, Pedro},
- journal={Computers \& Mathematics with Applications},
- volume={76},
- number={8},
- pages={1853--1862},
- year={2018},
- publisher={Elsevier}
-}
-
-@article{laizet2011incompact3d,
- title={Incompact3d: A powerful tool to tackle turbulence problems with up to O (105) computational cores},
- author={Laizet, Sylvain and Li, Ning},
- journal={International Journal for Numerical Methods in Fluids},
- volume={67},
- number={11},
- pages={1735--1757},
- year={2011},
- publisher={Wiley Online Library}
-}
+@ARTICLE{almgren1998conservative,
+ title={{A conservative adaptive projection method for the variable density incompressible Navier--Stokes equations}},
+ author={Almgren, Ann S and Bell, John B and Colella, Phillip and Howell, Louis H and Welcome, Michael L},
+ journal={Journal of Computational Physics},
+ volume={142},
+ number={1},
+ pages={1--46},
+ year={1998},
+ publisher={Elsevier},
+ DOI = {10.1006/jcph.1998.5890}
+}
+
+@ARTICLE{zeng2022parallel,
+ title={{A parallel cell-centered adaptive level set framework for efficient simulation of two-phase flows with subcycling and non-subcycling}},
+ author={Zeng, Yadong and Xuan, Anqing and Blaschke, Johannes and Shen, Lian},
+ journal={Journal of Computational Physics},
+ volume={448},
+ pages={110740},
+ year={2022},
+ publisher={Elsevier},
+ DOI = {10.1016/j.jcp.2021.110740}
+}
+
+@ARTICLE{zeng2023consistent,
+ title={{A consistent adaptive level set framework for incompressible two-phase flows with high density ratios and high Reynolds numbers}},
+ author={Zeng, Yadong and Liu, Han and Gao, Qiang and Almgren, Ann and Bhalla, Amneet Pal Singh and Shen, Lian},
+ journal={Journal of Computational Physics},
+ volume={478},
+ pages={111971},
+ year={2023},
+ publisher={Elsevier},
+ DOI = {10.1016/j.jcp.2023.111971}
+}
+
+@ARTICLE{li2024open,
+ title={{An open-source, adaptive solver for particle-resolved simulations with both subcycling and non-subcycling methods}},
+ author={Li, Xuzhu and Li, Chun and Li, Xiaokai and Li, Wenzhuo and Tang, Mingze and Zeng, Yadong and Zhu, Zhengping},
+ journal={Physics of Fluids},
+ volume={36},
+ number={11},
+ year={2024},
+ publisher={AIP Publishing},
+ DOI = {10.1063/5.0236509}
+}
+
+@ARTICLE{zhang2019amrex,
+ title={{AMReX: a framework for block-structured adaptive mesh refinement}},
+ author={Zhang, Weiqun and Almgren, Ann and Beckner, Vince and Bell, John and Blaschke, Johannes and Chan, Cy and Day, Marcus and Friesen, Brian and Gott, Kevin and Graves, Daniel and others},
+ journal={Journal of Open Source Software},
+ volume={4},
+ number={37},
+ year={2019},
+ publisher = {The Open Journal},
+ DOI = {10.21105/joss.01370}
+}
+
+@article{gong2023cp3d,
+ title={CP3d: A comprehensive Euler-Lagrange solver for direct numerical simulation of particle-laden flows},
+ author={Gong, Zheng and Wu, Zi and An, Chenge and Zhang, Bangwen and Fu, Xudong},
+ journal={Computer Physics Communications},
+ volume={286},
+ pages={108666},
+ year={2023},
+ publisher={Elsevier}
+}
+
+@article{costa2018fft,
+ title={A FFT-based finite-difference solver for massively-parallel direct numerical simulations of turbulent flows},
+ author={Costa, Pedro},
+ journal={Computers \& Mathematics with Applications},
+ volume={76},
+ number={8},
+ pages={1853--1862},
+ year={2018},
+ publisher={Elsevier}
+}
+
+@article{laizet2011incompact3d,
+ title={Incompact3d: A powerful tool to tackle turbulence problems with up to O (105) computational cores},
+ author={Laizet, Sylvain and Li, Ning},
+ journal={International Journal for Numerical Methods in Fluids},
+ volume={67},
+ number={11},
+ pages={1735--1757},
+ year={2011},
+ publisher={Wiley Online Library}
+}
diff --git a/JOSS_paper/paper.md b/JOSS_paper/paper.md
index 2ef86e25..293ff286 100644
--- a/JOSS_paper/paper.md
+++ b/JOSS_paper/paper.md
@@ -1,96 +1,96 @@
----
-title: 'IAMReX: an adaptive framework for the multiphase flow and fluid-particle interaction problems'
-tags:
- - C++
- - Computational Fluid Dynamics
- - Adaptive Mesh Refinement
- - Immersed Boundary Method
- - Multiphase Flow
-authors:
- - name: Chun Li
- orcid: 0009-0001-6081-5450
- affiliation: "1, 2"
- - name: Xuzhu Li
- orcid: 0009-0007-5348-4159
- affiliation: "1, 3"
- - name: Yiliang Wang
- orcid: 0000-0002-0156-2203
- affiliation: 4
- - name: Dewen Liu
- orcid: 0009-0006-2728-421X
- affiliation: 5
- - name: Shuai He
- orcid: 0009-0004-7754-8346
- affiliation: 6
- - name: Haoran Cheng
- orcid: 0009-0002-7636-038X
- affiliation: 7
- - name: Xiaokai Li
- orcid: 0009-0001-6639-0473
- affiliation: 8
- - name: Wenzhuo Li
- orcid: 0009-0001-7608-2992
- affiliation: 9
- - name: Mingze Tang
- orcid: 0009-0007-4194-9908
- affiliation: 10
- - name: Zhengping Zhu
- orcid: 0000-0002-1315-3554
- affiliation: 1
- - name: Yadong Zeng
- corresponding: true
- orcid: 0009-0001-7944-3597
- affiliation: 11
-affiliations:
- - name: Research Center for Astronomical Computing, Zhejiang Laboratory, Hangzhou, 311100, China
- index: 1
- - name: School of Energy and Power Engineering, Lanzhou University of Technology, Lanzhou, 730050, China
- index: 2
- - name: School of Mechanical Engineering, Hefei University of Technology, Hefei, 230009, China
- index: 3
- - name: Independent Researcher, Loveland, 45140, USA
- index: 4
- - name: School of Civil Engineering, Southwest Jiaotong University, Chengdu, 611756, China
- index: 5
- - name: School of Power and Energy, Northwestern Polytechnical University, Xi'an, 710129, China
- index: 6
- - name: Department of Electrical Engineering and Computer Science, University of Michigan, Ann Arbor, 48104, USA
- index: 7
- - name: School of Physical Science and Technology, ShanghaiTech University, Shanghai, 201210, China
- index: 8
- - name: Advanced Propulsion Laboratory, Department of Modern Mechanics, University of Science and Technology of China, Hefei, 230026, China
- index: 9
- - name: School of Aeronautics, Northwestern Polytechnical University, Xi'an, 710072, China
- index: 10
- - name: Department of Computer Science, University of Texas at Austin, Austin, 78712, USA
- index: 11
-
-date: 17 January 2025
-bibliography: paper.bib
-
-# # Optional fields if submitting to a ASS, AS journal too, see this blog post:
-# # https://blog.joss.theoj.org/2018/12/a-new-collaboration-with-aas-publishing
-# aas-doi: 10.3847/xxxxx <- update this with the DOI from ASS, AS once you know it.
-# aas-journal: Astrophysical Journal <- The name of the ASS, AS journal.
----
-
-# Summary
-
-IAMReX is an adaptive C++ solver designed for multiphase flow and fluid-particle interaction problems. It is built in a objected-oriented style and capable of high-performance massively parallel computing complex systems (e.g., gas-fluid interaction, cluster of particles).
-
-The original goal of IAMReX is to extend the capability of IAMR codes [@almgren1998conservative], which only uses a density-based solver to capture the diffused interface of the two-phase flow. IAMReX offers the Level Set (LS) method and the reinitialization techniques for accurately capturing the two-phase interface [@zeng2022parallel], which increases the robustness of simulations with high Reynolds number [@zeng2023consistent]. For fluid-particle interaction problems, IAMReX employs the multidirect forcing immersed boundary method [@li2024open]. The associated Lagrangian markers used to resolve fluid-particle interface only exist on the finest-level grid, which greatly reduces memory cost. Both the subcycling and non-subcycling time advancement methods are implemented, and these methods help to decouple the time advancement at different levels. In addition, IAMReX is a publicly accessible platform designed specifically for developing massively parallel block-structured adaptive mesh refinement (BSAMR) applications. The code now supports hybrid parallelization using either pure MPI or MPI & OpenMP for multicore machines with the help of the AMReX framework [@zhang2019amrex].
-
-The IAMReX code has undergone considerable development since 2023 and gained a few new contributors in the past two years. Although the projection-based flow solver is inherited from IAMR, IAMReX has added over 3,000 lines of new code, introduced 10 more new test cases, and contributed approximately 60 new commits on GitHub. The versatility, accuracy, and efficiency of the present IAMReX framework are demonstrated by simulating two-phase flow and fluid-particle interaction problems with various types of kinematic constraints. We carefully designed the document such that users can easily compile and run cases. Input files, profiling scripts, and raw postprocessing data are also available for reproducing all results.
-
-# Statement of need
-
-IAMReX is suitable for modeling multiphase flow problems and fluid-structure interaction problems. Its Level Set-based interface capturing technique can be beneficial for researchers studying phenomena such as wind over waves, breaking waves, and simulating the formation and disappearance of bubbles and droplets. Additionally, the immersed boundary method along with the collision models can parallelly resolve large-scale particles and capture their motions. Researchers working on studies of biological particle aggregation, sandstorms, wind erosion of ground surfaces, and seawater erosion of riverbeds are also among the target audience for this software.
-
-# State of the field
-We made great efforts to simulate more complex multiphase flows at higher resolution using IAMReX. One effort is to combine the AMR technique with the multidirect forcing immersed boundary method to resolve particles only on the finest-level grid. It significantly reduces the grid requirements for particle-resolved simulation compared with commonly used uniform grid solvers [Incompact3d](https://github.com/xcompact3d/Incompact3d), [CaNS](https://github.com/CaNS-World/CaNS), and [CP3d](https://github.com/GongZheng-Justin/CP3d). Additionally, we utilized a subcycling technique to alleviate the time step constraint on coarser levels. It minimizes the total time step needed by time advancement compared with the non-subcycling technique used in other AMR-related packages, such as [IBAMR](https://github.com/IBAMR/IBAMR.git), [basilisk](http://basilisk.fr/), and [incflo](https://github.com/AMReX-Fluids/incflo.git).
-
-# Acknowledgements
-
-C.L., X.L., Y.Z., and Z.Z. are grateful to Ann Almgren, John Bell, Andy Nonaka, Candace Gilet, Andrew Myers, Axel Huebl, Marc Day, and Weiqun Zhang in the Lawrence Berkeley National Laboratory (LBNL) for their discussions related to AMReX and IAMR. We sincerely thank all the developers who have contributed to the original IAMR repository, although this paper focus on the newly added features and does not include the names of all contributors. Y.Z. and Z.Z. also thank Prof. Lian Shen, Prof. Ruifeng Hu, and Prof. Xiaojing Zheng during their Ph.D. studies.
-
-# References
+---
+title: 'IAMReX: an adaptive framework for the multiphase flow and fluid-particle interaction problems'
+tags:
+ - C++
+ - Computational Fluid Dynamics
+ - Adaptive Mesh Refinement
+ - Immersed Boundary Method
+ - Multiphase Flow
+authors:
+ - name: Chun Li
+ orcid: 0009-0001-6081-5450
+ affiliation: "1, 2"
+ - name: Xuzhu Li
+ orcid: 0009-0007-5348-4159
+ affiliation: "1, 3"
+ - name: Yiliang Wang
+ orcid: 0000-0002-0156-2203
+ affiliation: 4
+ - name: Dewen Liu
+ orcid: 0009-0006-2728-421X
+ affiliation: 5
+ - name: Shuai He
+ orcid: 0009-0004-7754-8346
+ affiliation: 6
+ - name: Haoran Cheng
+ orcid: 0009-0002-7636-038X
+ affiliation: 7
+ - name: Xiaokai Li
+ orcid: 0009-0001-6639-0473
+ affiliation: 8
+ - name: Wenzhuo Li
+ orcid: 0009-0001-7608-2992
+ affiliation: 9
+ - name: Mingze Tang
+ orcid: 0009-0007-4194-9908
+ affiliation: 10
+ - name: Zhengping Zhu
+ orcid: 0000-0002-1315-3554
+ affiliation: 1
+ - name: Yadong Zeng
+ corresponding: true
+ orcid: 0009-0001-7944-3597
+ affiliation: 11
+affiliations:
+ - name: Research Center for Astronomical Computing, Zhejiang Laboratory, Hangzhou, 311100, China
+ index: 1
+ - name: School of Energy and Power Engineering, Lanzhou University of Technology, Lanzhou, 730050, China
+ index: 2
+ - name: School of Mechanical Engineering, Hefei University of Technology, Hefei, 230009, China
+ index: 3
+ - name: Independent Researcher, Loveland, 45140, USA
+ index: 4
+ - name: School of Civil Engineering, Southwest Jiaotong University, Chengdu, 611756, China
+ index: 5
+ - name: School of Power and Energy, Northwestern Polytechnical University, Xi'an, 710129, China
+ index: 6
+ - name: Department of Electrical Engineering and Computer Science, University of Michigan, Ann Arbor, 48104, USA
+ index: 7
+ - name: School of Physical Science and Technology, ShanghaiTech University, Shanghai, 201210, China
+ index: 8
+ - name: Advanced Propulsion Laboratory, Department of Modern Mechanics, University of Science and Technology of China, Hefei, 230026, China
+ index: 9
+ - name: School of Aeronautics, Northwestern Polytechnical University, Xi'an, 710072, China
+ index: 10
+ - name: Department of Computer Science, University of Texas at Austin, Austin, 78712, USA
+ index: 11
+
+date: 17 January 2025
+bibliography: paper.bib
+
+# # Optional fields if submitting to a ASS, AS journal too, see this blog post:
+# # https://blog.joss.theoj.org/2018/12/a-new-collaboration-with-aas-publishing
+# aas-doi: 10.3847/xxxxx <- update this with the DOI from ASS, AS once you know it.
+# aas-journal: Astrophysical Journal <- The name of the ASS, AS journal.
+---
+
+# Summary
+
+IAMReX is an adaptive C++ solver designed for multiphase flow and fluid-particle interaction problems. It is built in a objected-oriented style and capable of high-performance massively parallel computing complex systems (e.g., gas-fluid interaction, cluster of particles).
+
+The original goal of IAMReX is to extend the capability of IAMR codes [@almgren1998conservative], which only uses a density-based solver to capture the diffused interface of the two-phase flow. IAMReX offers the Level Set (LS) method and the reinitialization techniques for accurately capturing the two-phase interface [@zeng2022parallel], which increases the robustness of simulations with high Reynolds number [@zeng2023consistent]. For fluid-particle interaction problems, IAMReX employs the multidirect forcing immersed boundary method [@li2024open]. The associated Lagrangian markers used to resolve fluid-particle interface only exist on the finest-level grid, which greatly reduces memory cost. Both the subcycling and non-subcycling time advancement methods are implemented, and these methods help to decouple the time advancement at different levels. In addition, IAMReX is a publicly accessible platform designed specifically for developing massively parallel block-structured adaptive mesh refinement (BSAMR) applications. The code now supports hybrid parallelization using either pure MPI or MPI & OpenMP for multicore machines with the help of the AMReX framework [@zhang2019amrex].
+
+The IAMReX code has undergone considerable development since 2023 and gained a few new contributors in the past two years. Although the projection-based flow solver is inherited from IAMR, IAMReX has added over 3,000 lines of new code, introduced 10 more new test cases, and contributed approximately 60 new commits on GitHub. The versatility, accuracy, and efficiency of the present IAMReX framework are demonstrated by simulating two-phase flow and fluid-particle interaction problems with various types of kinematic constraints. We carefully designed the document such that users can easily compile and run cases. Input files, profiling scripts, and raw postprocessing data are also available for reproducing all results.
+
+# Statement of need
+
+IAMReX is suitable for modeling multiphase flow problems and fluid-structure interaction problems. Its Level Set-based interface capturing technique can be beneficial for researchers studying phenomena such as wind over waves, breaking waves, and simulating the formation and disappearance of bubbles and droplets. Additionally, the immersed boundary method along with the collision models can parallelly resolve large-scale particles and capture their motions. Researchers working on studies of biological particle aggregation, sandstorms, wind erosion of ground surfaces, and seawater erosion of riverbeds are also among the target audience for this software.
+
+# State of the field
+We made great efforts to simulate more complex multiphase flows at higher resolution using IAMReX. One effort is to combine the AMR technique with the multidirect forcing immersed boundary method to resolve particles only on the finest-level grid. It significantly reduces the grid requirements for particle-resolved simulation compared with commonly used uniform grid solvers [Incompact3d](https://github.com/xcompact3d/Incompact3d), [CaNS](https://github.com/CaNS-World/CaNS), and [CP3d](https://github.com/GongZheng-Justin/CP3d). Additionally, we utilized a subcycling technique to alleviate the time step constraint on coarser levels. It minimizes the total time step needed by time advancement compared with the non-subcycling technique used in other AMR-related packages, such as [IBAMR](https://github.com/IBAMR/IBAMR.git), [basilisk](http://basilisk.fr/), and [incflo](https://github.com/AMReX-Fluids/incflo.git).
+
+# Acknowledgements
+
+C.L., X.L., Y.Z., and Z.Z. are grateful to Ann Almgren, John Bell, Andy Nonaka, Candace Gilet, Andrew Myers, Axel Huebl, Marc Day, and Weiqun Zhang in the Lawrence Berkeley National Laboratory (LBNL) for their discussions related to AMReX and IAMR. We sincerely thank all the developers who have contributed to the original IAMR repository, although this paper focus on the newly added features and does not include the names of all contributors. Y.Z. and Z.Z. also thank Prof. Lian Shen, Prof. Ruifeng Hu, and Prof. Xiaojing Zheng during their Ph.D. studies.
+
+# References
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 1736e9dd..00000000
--- a/LICENSE
+++ /dev/null
@@ -1,29 +0,0 @@
-BSD 3-Clause License
-
-Copyright (c) 2018,
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this
- list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice,
- this list of conditions and the following disclaimer in the documentation
- and/or other materials provided with the distribution.
-
-* Neither the name of the copyright holder nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt
new file mode 100644
index 00000000..b51e7d67
--- /dev/null
+++ b/LICENSES/BSD-3-Clause.txt
@@ -0,0 +1,12 @@
+Copyright (c) 2022-2025
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/OpenSource.txt b/LICENSES/LicenseRef-OpenSource.txt
similarity index 98%
rename from OpenSource.txt
rename to LICENSES/LicenseRef-OpenSource.txt
index 725199eb..1f22ddf2 100644
--- a/OpenSource.txt
+++ b/LICENSES/LicenseRef-OpenSource.txt
@@ -1,176 +1,176 @@
-SOURCE CODE LICENSE AGREEMENT
-Software: IAMR
-Version: Oct. 12, 2000 Release
-
-IMPORTANT - READ CAREFULLY: This License Agreement ("Agreement") is a
-legal agreement between you (in your capacity as an individual and as
-an agent for your company, institution or other entity) and The
-Regents of the University of California, Department of Energy
-contract-operators of the Ernest Orlando Lawrence Berkeley National
-Laboratory ("Berkeley Lab"). Downloading, installing, using, or
-copying of the Software (as defined below) by you or by a third party
-on your behalf indicates your agreement to be bound by the terms and
-conditions of this Agreement. If you do not agree to these terms and
-conditions, do not download, install or use the Software.
-
-1. LICENSE GRANT. Berkeley Lab grants you, and you hereby accept, a
- non-exclusive, royalty-free perpetual license to install, use,
- modify, prepare derivative works, incorporate into other computer
- software, and distribute the version noted above of the computer
- software program noted above, in binary and source code format, or
- any derivative work thereof, together with any associated media,
- printed materials, and on-line or electronic documentation (if
- any) provided by Berkeley Lab (collectively, the "Software"),
- subject to the following terms and conditions: (i) any
- distribution of the Software shall bind the receiver to the terms
- and conditions of this Agreement; (ii) any distribution of the
- Software in modified form shall clearly state that the Software
- has been modified from the version originally obtained from
- Berkeley Lab. This version of the Software constitutes a research
- prototype and may be changed substantially. The license grant set
- forth above is subject to receipt by Berkeley Lab of any required
- U.S. Department of Energy approvals.
-
-2. COPYRIGHT; RETENTION OF RIGHTS. The above license grant is
- conditioned on the following: (i) you must reproduce all copyright
- notices and other proprietary notices on any copies of the
- Software and you must not remove such notices; (ii) in the event
- you compile the Software, you will include the copyright notice
- with the binary in such a manner as to allow it to be easily
- viewable; (iii) if you incorporate the Software into other code,
- you must provide notice that the code contains the Software and
- include a copy of the copyright notices and other proprietary
- notices. All copies of the Software shall be subject to the terms
- of this Agreement. Subject to approval by the U.S. Department of
- Energy: (a) you hereby acknowledge that the Software is protected
- by United States copyright law and international treaty
- provisions; (b) Berkeley Lab, and its licensors (if any), hereby
- reserve all rights in the Software which are not explicitly
- granted to you herein; (c) without limiting the generality of the
- foregoing, Berkeley Lab and its licensors retain all title,
- copyright, and other proprietary interests in the Software and any
- copies thereof, and you do not acquire any rights, express or
- implied, in the Software, other than those specifically set forth
- in this Agreement.
-
-3. NO MAINTENANCE OR SUPPORT; TREATMENT OF ENHANCEMENTS YOU CHOOSE TO
- PROVIDE TO BERKELEY LAB. Berkeley Lab is under no obligation
- whatsoever to: (i) provide maintenance or support for the
- Software; or (ii) to notify you of bug fixes, patches, or upgrades
- to the features, functionality or performance of the Software
- ("Enhancements") (if any), whether developed by Berkeley Lab or
- third parties. If, in its sole discretion, Berkeley Lab makes an
- Enhancement available to you and Berkeley Lab does not separately
- enter into a written license agreement with you relating to such
- bug fix, patch or upgrade, then it shall be deemed incorporated
- into the Software and subject to this Agreement. You are under no
- obligation whatsoever to provide any Enhancements to Berkeley Lab
- that you may develop over time; however, if you choose to provide
- Berkeley Lab with Enhancements in source code form that you have
- developed without contemporaneously requiring Berkeley Lab to
- enter into a separate written license agreement, then you hereby
- grant Berkeley Lab a non-exclusive, royalty-free perpetual license
- to install, use, modify, prepare derivative works, incorporate
- into the Software or other computer software, distribute, and
- sublicense your Enhancements or derivative works thereof, in
- binary and source code form.
-
-4. U.S. GOVERNMENT RIGHTS. The Software was developed under funding
- from the U.S. Department of Energy and the U.S. Government
- consequently retains certain rights as follows: the
- U.S. Government has been granted for itself and others acting on
- its behalf a paid-up, nonexclusive, irrevocable, worldwide license
- in the Software to reproduce, prepare derivative works, and
- perform publicly and display publicly. Beginning five (5) years
- after the date permission to assert copyright was granted by the
- U.S. Dept. of Energy, and subject to any subsequent five (5) year
- renewals, the U.S. Government is granted for itself and others
- acting on its behalf a paid-up, nonexclusive, irrevocable,
- worldwide license in the Software to reproduce, prepare derivative
- works, distribute copies to the public, perform publicly and
- display publicly, and to permit others to do so.
-
-5. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT
- WARRANTY OF ANY KIND. BERKELEY LAB, ITS LICENSORS, THE UNITED
- STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND THEIR
- EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR IMPLIED,
- INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR
- NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY OR
- RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF
- THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE SOFTWARE WOULD
- NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO NOT WARRANT THAT THE
- SOFTWARE WILL FUNCTION UNINTERRUPTED, THAT IT IS ERROR-FREE OR
- THAT ANY ERRORS WILL BE CORRECTED.
-
-6. LIMITATION OF LIABILITY. IN NO EVENT WILL BERKELEY LAB OR ITS
- LICENSORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL, CONSEQUENTIAL,
- SPECIAL OR PUNITIVE DAMAGES OF ANY KIND OR NATURE, INCLUDING BUT
- NOT LIMITED TO LOSS OF PROFITS OR LOSS OF DATA, FOR ANY REASON
- WHATSOEVER, WHETHER SUCH LIABILITY IS ASSERTED ON THE BASIS OF
- CONTRACT, TORT (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR
- OTHERWISE, EVEN IF BERKELEY LAB HAS BEEN WARNED OF THE POSSIBILITY
- OF SUCH LOSS OR DAMAGES. IN NO EVENT SHALL BERKELEY LAB'S
- LIABILITY FOR DAMAGES ARISING FROM OR IN CONNECTION WITH THIS
- AGREEMENT EXCEED THE AMOUNT PAID BY YOU FOR THE SOFTWARE.
-
-7. INDEMNITY. You shall indemnify, defend, and hold harmless
- Berkeley Lab, the U.S. Government, the Software developers, the
- Software sponsors, and their agents, officers, and employees,
- against any and all claims, suits, losses, damage, costs, fees,
- and expenses arising out of or in connection with this Agreement.
- You shall pay all costs incurred by Berkeley Lab in enforcing this
- provision, including reasonable attorney fees.
-
-8. TERM AND TERMINATION. The license granted to you under this
- Agreement will continue perpetually unless terminated by Berkeley
- Lab in accordance with this Agreement. If you breach any term of
- this Agreement, and fail to cure such breach within thirty (30)
- days of the date of written notice, this Agreement shall
- immediately terminate. Upon such termination, you shall
- immediately cease using the Software, return to Berkeley Lab, or
- destroy, all copies of the Software, and provide Berkeley Lab with
- written certification of your compliance with the foregoing.
- Termination shall not relieve you from your obligations arising
- prior to such termination. Notwithstanding any provision of this
- Agreement to the contrary, Sections 5 through 10 shall survive
- termination of this Agreement.
-
-9. EXPORT CONTROLS. You shall observe all applicable United States
- and foreign laws and regulations (if any) with respect to the
- export, re-export, diversion or transfer of the Software, related
- technical data and direct products thereof, including, without
- limitation, the International Traffic in Arms Regulations (ITAR)
- and the Export Administration Regulations.
-
-10. NO ENDORSEMENT. In accordance with California Education Code
- Section 92000, you shall not use in advertising, publicity or
- other promotional activities any name, trade name, trademark, or
- other designation of the University of California, nor shall you
- so use "Ernest Orlando Lawrence Berkeley National Laboratory" or
- "United States Department of Energy" (including any contraction,
- abbreviation, or simulation of any of the foregoing) without
- Berkeley Lab's prior written consent.
-
-11. GENERAL. This Agreement shall be governed by the laws of the
- State of California, excluding its rules governing conflicts of
- laws. No provision in either party's purchase orders, or in any
- other business forms employed by either party will supersede the
- terms of this Agreement, and no modification or amendment of this
- Agreement is binding, unless in writing signed by a duly
- authorized representative of each party. This Agreement is
- binding upon and shall inure to the benefit of Berkeley Lab, its
- successors and assigns. This Agreement represents the entire
- understanding of the parties, and supersedes all previous
- communications, written or oral, relating to the subject of this
- Agreement. If you have any questions concerning this license,
- contact Lawrence Berkeley National Laboratory, Technology Transfer
- Department, One Cyclotron Road, MS 90-1070, Berkeley, CA 94720,
- Attn: Software Licensing or via e-mail at ipo@lbl.gov.
-
-If you have any questions or feedback concerning this Software,
-contact the Center for Computational Sciences and Engineering,
-Lawrence Berkeley National Laboratory, One Cyclotron Road, MS
-50A-3111, Berkeley, CA 94720 or via email at ASAlmgren@lbl.gov
-
-Form rev000928
+SOURCE CODE LICENSE AGREEMENT
+Software: IAMR
+Version: Oct. 12, 2000 Release
+
+IMPORTANT - READ CAREFULLY: This License Agreement ("Agreement") is a
+legal agreement between you (in your capacity as an individual and as
+an agent for your company, institution or other entity) and The
+Regents of the University of California, Department of Energy
+contract-operators of the Ernest Orlando Lawrence Berkeley National
+Laboratory ("Berkeley Lab"). Downloading, installing, using, or
+copying of the Software (as defined below) by you or by a third party
+on your behalf indicates your agreement to be bound by the terms and
+conditions of this Agreement. If you do not agree to these terms and
+conditions, do not download, install or use the Software.
+
+1. LICENSE GRANT. Berkeley Lab grants you, and you hereby accept, a
+ non-exclusive, royalty-free perpetual license to install, use,
+ modify, prepare derivative works, incorporate into other computer
+ software, and distribute the version noted above of the computer
+ software program noted above, in binary and source code format, or
+ any derivative work thereof, together with any associated media,
+ printed materials, and on-line or electronic documentation (if
+ any) provided by Berkeley Lab (collectively, the "Software"),
+ subject to the following terms and conditions: (i) any
+ distribution of the Software shall bind the receiver to the terms
+ and conditions of this Agreement; (ii) any distribution of the
+ Software in modified form shall clearly state that the Software
+ has been modified from the version originally obtained from
+ Berkeley Lab. This version of the Software constitutes a research
+ prototype and may be changed substantially. The license grant set
+ forth above is subject to receipt by Berkeley Lab of any required
+ U.S. Department of Energy approvals.
+
+2. COPYRIGHT; RETENTION OF RIGHTS. The above license grant is
+ conditioned on the following: (i) you must reproduce all copyright
+ notices and other proprietary notices on any copies of the
+ Software and you must not remove such notices; (ii) in the event
+ you compile the Software, you will include the copyright notice
+ with the binary in such a manner as to allow it to be easily
+ viewable; (iii) if you incorporate the Software into other code,
+ you must provide notice that the code contains the Software and
+ include a copy of the copyright notices and other proprietary
+ notices. All copies of the Software shall be subject to the terms
+ of this Agreement. Subject to approval by the U.S. Department of
+ Energy: (a) you hereby acknowledge that the Software is protected
+ by United States copyright law and international treaty
+ provisions; (b) Berkeley Lab, and its licensors (if any), hereby
+ reserve all rights in the Software which are not explicitly
+ granted to you herein; (c) without limiting the generality of the
+ foregoing, Berkeley Lab and its licensors retain all title,
+ copyright, and other proprietary interests in the Software and any
+ copies thereof, and you do not acquire any rights, express or
+ implied, in the Software, other than those specifically set forth
+ in this Agreement.
+
+3. NO MAINTENANCE OR SUPPORT; TREATMENT OF ENHANCEMENTS YOU CHOOSE TO
+ PROVIDE TO BERKELEY LAB. Berkeley Lab is under no obligation
+ whatsoever to: (i) provide maintenance or support for the
+ Software; or (ii) to notify you of bug fixes, patches, or upgrades
+ to the features, functionality or performance of the Software
+ ("Enhancements") (if any), whether developed by Berkeley Lab or
+ third parties. If, in its sole discretion, Berkeley Lab makes an
+ Enhancement available to you and Berkeley Lab does not separately
+ enter into a written license agreement with you relating to such
+ bug fix, patch or upgrade, then it shall be deemed incorporated
+ into the Software and subject to this Agreement. You are under no
+ obligation whatsoever to provide any Enhancements to Berkeley Lab
+ that you may develop over time; however, if you choose to provide
+ Berkeley Lab with Enhancements in source code form that you have
+ developed without contemporaneously requiring Berkeley Lab to
+ enter into a separate written license agreement, then you hereby
+ grant Berkeley Lab a non-exclusive, royalty-free perpetual license
+ to install, use, modify, prepare derivative works, incorporate
+ into the Software or other computer software, distribute, and
+ sublicense your Enhancements or derivative works thereof, in
+ binary and source code form.
+
+4. U.S. GOVERNMENT RIGHTS. The Software was developed under funding
+ from the U.S. Department of Energy and the U.S. Government
+ consequently retains certain rights as follows: the
+ U.S. Government has been granted for itself and others acting on
+ its behalf a paid-up, nonexclusive, irrevocable, worldwide license
+ in the Software to reproduce, prepare derivative works, and
+ perform publicly and display publicly. Beginning five (5) years
+ after the date permission to assert copyright was granted by the
+ U.S. Dept. of Energy, and subject to any subsequent five (5) year
+ renewals, the U.S. Government is granted for itself and others
+ acting on its behalf a paid-up, nonexclusive, irrevocable,
+ worldwide license in the Software to reproduce, prepare derivative
+ works, distribute copies to the public, perform publicly and
+ display publicly, and to permit others to do so.
+
+5. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT
+ WARRANTY OF ANY KIND. BERKELEY LAB, ITS LICENSORS, THE UNITED
+ STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND THEIR
+ EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR
+ NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY OR
+ RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF
+ THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF THE SOFTWARE WOULD
+ NOT INFRINGE PRIVATELY OWNED RIGHTS, (4) DO NOT WARRANT THAT THE
+ SOFTWARE WILL FUNCTION UNINTERRUPTED, THAT IT IS ERROR-FREE OR
+ THAT ANY ERRORS WILL BE CORRECTED.
+
+6. LIMITATION OF LIABILITY. IN NO EVENT WILL BERKELEY LAB OR ITS
+ LICENSORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL, CONSEQUENTIAL,
+ SPECIAL OR PUNITIVE DAMAGES OF ANY KIND OR NATURE, INCLUDING BUT
+ NOT LIMITED TO LOSS OF PROFITS OR LOSS OF DATA, FOR ANY REASON
+ WHATSOEVER, WHETHER SUCH LIABILITY IS ASSERTED ON THE BASIS OF
+ CONTRACT, TORT (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR
+ OTHERWISE, EVEN IF BERKELEY LAB HAS BEEN WARNED OF THE POSSIBILITY
+ OF SUCH LOSS OR DAMAGES. IN NO EVENT SHALL BERKELEY LAB'S
+ LIABILITY FOR DAMAGES ARISING FROM OR IN CONNECTION WITH THIS
+ AGREEMENT EXCEED THE AMOUNT PAID BY YOU FOR THE SOFTWARE.
+
+7. INDEMNITY. You shall indemnify, defend, and hold harmless
+ Berkeley Lab, the U.S. Government, the Software developers, the
+ Software sponsors, and their agents, officers, and employees,
+ against any and all claims, suits, losses, damage, costs, fees,
+ and expenses arising out of or in connection with this Agreement.
+ You shall pay all costs incurred by Berkeley Lab in enforcing this
+ provision, including reasonable attorney fees.
+
+8. TERM AND TERMINATION. The license granted to you under this
+ Agreement will continue perpetually unless terminated by Berkeley
+ Lab in accordance with this Agreement. If you breach any term of
+ this Agreement, and fail to cure such breach within thirty (30)
+ days of the date of written notice, this Agreement shall
+ immediately terminate. Upon such termination, you shall
+ immediately cease using the Software, return to Berkeley Lab, or
+ destroy, all copies of the Software, and provide Berkeley Lab with
+ written certification of your compliance with the foregoing.
+ Termination shall not relieve you from your obligations arising
+ prior to such termination. Notwithstanding any provision of this
+ Agreement to the contrary, Sections 5 through 10 shall survive
+ termination of this Agreement.
+
+9. EXPORT CONTROLS. You shall observe all applicable United States
+ and foreign laws and regulations (if any) with respect to the
+ export, re-export, diversion or transfer of the Software, related
+ technical data and direct products thereof, including, without
+ limitation, the International Traffic in Arms Regulations (ITAR)
+ and the Export Administration Regulations.
+
+10. NO ENDORSEMENT. In accordance with California Education Code
+ Section 92000, you shall not use in advertising, publicity or
+ other promotional activities any name, trade name, trademark, or
+ other designation of the University of California, nor shall you
+ so use "Ernest Orlando Lawrence Berkeley National Laboratory" or
+ "United States Department of Energy" (including any contraction,
+ abbreviation, or simulation of any of the foregoing) without
+ Berkeley Lab's prior written consent.
+
+11. GENERAL. This Agreement shall be governed by the laws of the
+ State of California, excluding its rules governing conflicts of
+ laws. No provision in either party's purchase orders, or in any
+ other business forms employed by either party will supersede the
+ terms of this Agreement, and no modification or amendment of this
+ Agreement is binding, unless in writing signed by a duly
+ authorized representative of each party. This Agreement is
+ binding upon and shall inure to the benefit of Berkeley Lab, its
+ successors and assigns. This Agreement represents the entire
+ understanding of the parties, and supersedes all previous
+ communications, written or oral, relating to the subject of this
+ Agreement. If you have any questions concerning this license,
+ contact Lawrence Berkeley National Laboratory, Technology Transfer
+ Department, One Cyclotron Road, MS 90-1070, Berkeley, CA 94720,
+ Attn: Software Licensing or via e-mail at ipo@lbl.gov.
+
+If you have any questions or feedback concerning this Software,
+contact the Center for Computational Sciences and Engineering,
+Lawrence Berkeley National Laboratory, One Cyclotron Road, MS
+50A-3111, Berkeley, CA 94720 or via email at ASAlmgren@lbl.gov
+
+Form rev000928
diff --git a/README.md b/README.md
index b73037ff..97505a64 100644
--- a/README.md
+++ b/README.md
@@ -1,223 +1,223 @@
-
-

-
-
-
-
-
-
-
-[Overview](#Overview) -
-[Features](#Features) -
-[Examples](#Examples) -
-[Install](#Install) -
-[Tools](#Tools) -
-[Acknowledgements](#Acknowledgements) -
-[Contact](#Contact)
-
-
-
-
-## Overview
-
-This IAMReX repo extends the capability of original [IAMR](https://github.com/AMReX-Fluids/IAMR) codes, aiming at simulating the multiphase incompressible flows and fluid structure interaction problems on both CPUs and GPUs with/without subcycling. The Navier-Stokes equations are solved on an adaptive semi-staggered grid using the projection method. The gas-liquid interface is captured using the level set (LS) method. The fluid-solid interface is resolved using the multidirect forcing immersed boundary method (IBM). The particle-wall as well as the particle-particle collisions are also captured by the adaptive collision time model (ACTM).
-
-
-
-## Features
-
-- LS method and reinitialization schemes
-- Multidirect forcing Immersed Boundary Method
-- Particle Collision Algorithms with Discrete Element Method
-
-
-
-## Examples
-
-- [Reversed Single Vortex (RSV)](./Tutorials/RSV/)
-
-
-

-
-
Profiles of drop interface in the RSV problem at t/T=1 after one rotation. Black line: Analytical Solution; Red line: 64*64; Blue line: 128*128; Green line: 256*256
-
-
-
-
-- [Rayleigh-Taylor (RT) instability](./Tutorials/RayleighTaylor_LS/)
-
-
-
-
-

-
-
(a) Density profile at t/T=2.42 using LS method. (b) Density profile at t/T=2.42 using IAMR convective scheme.
-
-
-
-
-
-

-
-
Comparison of the tip locations of the falling fluid and the rising fluid.
-
-
-
-
-
-
-- [Cluster of monodisperse particles](./Tutorials/Monodisperse/)
-
-
-

-
-
Contours of velocity magnitude in y − z plane
-
-
-
-
-
-
-## Install
-
-### Download
-
-Our codes rely on the following packages:
-
-1. AMReX:`git clone https://github.com/ruohai0925/amrex`
-2. AMReX-Hydro:`git clone https://github.com/ruohai0925/AMReX-Hydro`
-3. This repo:`git clone https://github.com/ruohai0925/IAMReX/tree/development`
-
-After cloning, one will have three folder in your current directory:`AMReX`,`AMReX-Hydro`,`IAMReX`.
-
-### Compile
-
-We recommend using the GNU compiler to compile the program on the Linux platform. The compilation process requires preparing a `make` file, which you can find in the example folder under Tutorials. It is strongly recommended to use the `GNUmakefile` prepared in advance by the example. If you want to know more about the configuration information of the compilation parameters, you can check it in the [AMReX building](https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html).
-
-For example, if we want to compile in the `FlowPastSphere` , refer to the following steps:
-
-1. `cd` to the FlowPastSphere directory
-
- ```shell
- cd IAMReX/Tutorials/FlowPastSphere
- ```
-
-2. Modify compilation parameters in `GNUmakefile`.
-
- The compilation parameters depend on your computing platform. If you use `MPI` to run your program, then set `USE_MPI = TRUE`. If you are running the program with `Nvidia GPU(CUDA)`, then set `USE_CUDA = TRUE`. When using GPU runtime, please make sure your CUDA environment is ok.
-
-3. Compile
-
- With the above settings, you can compile the program:
-
- ```shell
- make
- ```
-
- You can add parameters after `make`, such as `-f your_make_file` to customize your parameter file, and `-j 8` to specify `8` threads to participate in the compilation.
-
- If the compilation is successful, an executable file will be generated. Usually the executable file is named in the following format:`amr[DIM].[Compiler manufacturers].[computing platform].[is Debug].ex`. The executable file name of the Release version of the three-dimensional compiled using the GNU compiler in the MPI environment is: `amr3d.GNU.MPI.ex`.
-
-
-### Usage
-
-You should takes an input file as its first command-line argument. The file may contain a set of parameter definitions that will overrides defaults set in the code. In the `FlowPastSphere`, you can find a file named `inputs.3d.flow_past_sphere`, run:
-
-```shell
-./amr3d.GNU.MPI.ex inputs.3d.flow_past_sphere
-```
-
-If you use `MPI` to run your program, you can type:
-
-```shell
-mpirun -np how_many_threads amr3d.GNU.MPI.ex inputs.3d.flow_past_sphere
-```
-
-This code typically generates subfolders in the current folder that are named `plt00000`, `plt00010`, etc, and `chk00000`, `chk00010`, etc. These are called plotfiles and checkpoint files. The plotfiles are used for visualization of derived fields; the checkpoint files are used for restarting the code.
-
-
-
-## Tools
-
-In the `Tools` folder, there are some post-processing script for DIBM. The `postProcess` submodule is used to process the generated particle data file. The specific usage can be seen in the `README`.
-
-In addition, a Fortran code for generating a particle bed is provided, You can customize the domain size and particle size you want.
-
-```fortran
-! Line 10 - 15
-! domain scheme
-LX_DOMAIN=3.0D0*Pi
-LY_DOMAIN=6.0D0*Pi
-LZ_DOMAIN=3.0D0*Pi
-! D
-SPD=1.D0
-```
-
-Compile the code and run it, and a file named `SPC.DAT` will be generated, in which each line represents the position of a particle.
-
-```shell
-gfortran -o fixBed DomainFill.F90
-./fixBed
-```
-
-## State of the field
-We made great efforts to simulate more complex multiphase flows at higher resolution using IAMReX. One effort is to combine the AMR technique with the multidirect forcing immersed boundary method to resolve particles only on the finest-level grid. It significantly reduces the grid requirements for particle-resolved simulation compared with commonly used uniform grid solvers [Incompact3d](https://github.com/xcompact3d/Incompact3d), [CaNS](https://github.com/CaNS-World/CaNS), and [CP3d](https://github.com/GongZheng-Justin/CP3d). Additionally, we utilized a subcycling technique to alleviate the time step constraint on coarser levels. It minimizes the total time step needed by time advancement compared with the non-subcycling technique used in other AMR-related packages, such as [IBAMR](https://github.com/IBAMR/IBAMR.git), [basilisk](http://basilisk.fr/), and [incflo](https://github.com/AMReX-Fluids/incflo.git).
-
-## Get Help
-
-You can also view questions
-and ask your own on our [GitHub Discussions](https://github.com/ruohai0925/IAMReX/discussions) page.
-To obtain additional help, simply post an issue.
-
-## Contribute
-
-We are always happy to have users contribute to the IAMReX source code. To
-contribute, issue a pull request against the development branch.
-Any level of changes are welcomed: documentation, bug fixes, new test problems,
-new solvers, etc. For more details on how to contribute to IAMReX, please see
-[CONTRIBUTING.md](CONTRIBUTING.md).
-
-💡 If you're using IAMReX in your own GitHub projects, consider adding `IAMReX`
-as a [repository topic](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/classifying-your-repository-with-topics)!
-This helps others discover related work and strengthens the IAMReX ecosystem.
-
-## Citation
-
-To cite IAMReX, please use
-
-```
-@article{10.1063/5.0236509,
- author = {Li, Xuzhu (李虚竹) and Li, Chun (李春) and Li, Xiaokai (李晓凯) and Li, Wenzhuo (李文卓) and Tang, Mingze (唐铭泽) and Zeng, Yadong (曾亚东) and Zhu, Zhengping (朱正平)},
- title = {An open-source, adaptive solver for particle-resolved simulations with both subcycling and non-subcycling methods},
- journal = {Physics of Fluids},
- volume = {36},
- number = {11},
- pages = {113335},
- year = {2024},
- month = {11},
- issn = {1070-6631},
- doi = {10.1063/5.0236509},
- url = {https://doi.org/10.1063/5.0236509},
- eprint = {https://pubs.aip.org/aip/pof/article-pdf/doi/10.1063/5.0236509/20247663/113335\_1\_5.0236509.pdf},
-}
-
-@inbook{doi:10.2514/6.2025-1865,
- author = {Dewen Liu and Shuai He and Haoran Cheng and Yadong Zeng},
- title = {Investigate the Efficiency of Incompressible Flow Simulations on CPUs and GPUs With BSAMR},
- booktitle = {AIAA SCITECH 2025 Forum},
- doi = {10.2514/6.2025-1865},
- URL = {https://arc.aiaa.org/doi/abs/10.2514/6.2025-1865},
- eprint = {https://arc.aiaa.org/doi/pdf/10.2514/6.2025-1865},
-}
-
-```
-
-## Acknowledgements
-
-We are grateful to Ann Almgren, Andy Nonaka, Andrew Myers, Axel Huebl, Marc Day, and Weiqun Zhang in the Lawrence Berkeley National Laboratory (LBNL) for their discussions related to [AMReX](https://github.com/AMReX-Codes/amrex) and [IAMR](https://github.com/AMReX-Fluids/IAMR). Y.Z. and Z.Z. also thank Prof. Lian Shen, Prof. Ruifeng Hu, and Prof. Xiaojing Zheng during their Ph.D. studies.
-
-
-
-## Contact
-
-If you have any question or wanna contribute to the code, please don't hesitate to contact us via the [GitHub Issues](https://github.com/ruohai0925/IAMReX/issues) or zdsjtu@gmail.com.
+
+

+
+
+
+
+
+
+
+[Overview](#Overview) -
+[Features](#Features) -
+[Examples](#Examples) -
+[Install](#Install) -
+[Tools](#Tools) -
+[Acknowledgements](#Acknowledgements) -
+[Contact](#Contact)
+
+
+
+
+## Overview
+
+This IAMReX repo extends the capability of original [IAMR](https://github.com/AMReX-Fluids/IAMR) codes, aiming at simulating the multiphase incompressible flows and fluid structure interaction problems on both CPUs and GPUs with/without subcycling. The Navier-Stokes equations are solved on an adaptive semi-staggered grid using the projection method. The gas-liquid interface is captured using the level set (LS) method. The fluid-solid interface is resolved using the multidirect forcing immersed boundary method (IBM). The particle-wall as well as the particle-particle collisions are also captured by the adaptive collision time model (ACTM).
+
+
+
+## Features
+
+- LS method and reinitialization schemes
+- Multidirect forcing Immersed Boundary Method
+- Particle Collision Algorithms with Discrete Element Method
+
+
+
+## Examples
+
+- [Reversed Single Vortex (RSV)](./Tutorials/RSV/)
+
+
+

+
+
Profiles of drop interface in the RSV problem at t/T=1 after one rotation. Black line: Analytical Solution; Red line: 64*64; Blue line: 128*128; Green line: 256*256
+
+
+
+
+- [Rayleigh-Taylor (RT) instability](./Tutorials/RayleighTaylor_LS/)
+
+
+
+
+

+
+
(a) Density profile at t/T=2.42 using LS method. (b) Density profile at t/T=2.42 using IAMR convective scheme.
+
+
+
+
+
+

+
+
Comparison of the tip locations of the falling fluid and the rising fluid.
+
+
+
+
+
+
+- [Cluster of monodisperse particles](./Tutorials/Monodisperse/)
+
+
+

+
+
Contours of velocity magnitude in y − z plane
+
+
+
+
+
+
+## Install
+
+### Download
+
+Our codes rely on the following packages:
+
+1. AMReX:`git clone https://github.com/ruohai0925/amrex`
+2. AMReX-Hydro:`git clone https://github.com/ruohai0925/AMReX-Hydro`
+3. This repo:`git clone https://github.com/ruohai0925/IAMReX/tree/development`
+
+After cloning, one will have three folder in your current directory:`AMReX`,`AMReX-Hydro`,`IAMReX`.
+
+### Compile
+
+We recommend using the GNU compiler to compile the program on the Linux platform. The compilation process requires preparing a `make` file, which you can find in the example folder under Tutorials. It is strongly recommended to use the `GNUmakefile` prepared in advance by the example. If you want to know more about the configuration information of the compilation parameters, you can check it in the [AMReX building](https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html).
+
+For example, if we want to compile in the `FlowPastSphere` , refer to the following steps:
+
+1. `cd` to the FlowPastSphere directory
+
+ ```shell
+ cd IAMReX/Tutorials/FlowPastSphere
+ ```
+
+2. Modify compilation parameters in `GNUmakefile`.
+
+ The compilation parameters depend on your computing platform. If you use `MPI` to run your program, then set `USE_MPI = TRUE`. If you are running the program with `Nvidia GPU(CUDA)`, then set `USE_CUDA = TRUE`. When using GPU runtime, please make sure your CUDA environment is ok.
+
+3. Compile
+
+ With the above settings, you can compile the program:
+
+ ```shell
+ make
+ ```
+
+ You can add parameters after `make`, such as `-f your_make_file` to customize your parameter file, and `-j 8` to specify `8` threads to participate in the compilation.
+
+ If the compilation is successful, an executable file will be generated. Usually the executable file is named in the following format:`amr[DIM].[Compiler manufacturers].[computing platform].[is Debug].ex`. The executable file name of the Release version of the three-dimensional compiled using the GNU compiler in the MPI environment is: `amr3d.GNU.MPI.ex`.
+
+
+### Usage
+
+You should takes an input file as its first command-line argument. The file may contain a set of parameter definitions that will overrides defaults set in the code. In the `FlowPastSphere`, you can find a file named `inputs.3d.flow_past_sphere`, run:
+
+```shell
+./amr3d.GNU.MPI.ex inputs.3d.flow_past_sphere
+```
+
+If you use `MPI` to run your program, you can type:
+
+```shell
+mpirun -np how_many_threads amr3d.GNU.MPI.ex inputs.3d.flow_past_sphere
+```
+
+This code typically generates subfolders in the current folder that are named `plt00000`, `plt00010`, etc, and `chk00000`, `chk00010`, etc. These are called plotfiles and checkpoint files. The plotfiles are used for visualization of derived fields; the checkpoint files are used for restarting the code.
+
+
+
+## Tools
+
+In the `Tools` folder, there are some post-processing script for DIBM. The `postProcess` submodule is used to process the generated particle data file. The specific usage can be seen in the `README`.
+
+In addition, a Fortran code for generating a particle bed is provided, You can customize the domain size and particle size you want.
+
+```fortran
+! Line 10 - 15
+! domain scheme
+LX_DOMAIN=3.0D0*Pi
+LY_DOMAIN=6.0D0*Pi
+LZ_DOMAIN=3.0D0*Pi
+! D
+SPD=1.D0
+```
+
+Compile the code and run it, and a file named `SPC.DAT` will be generated, in which each line represents the position of a particle.
+
+```shell
+gfortran -o fixBed DomainFill.F90
+./fixBed
+```
+
+## State of the field
+We made great efforts to simulate more complex multiphase flows at higher resolution using IAMReX. One effort is to combine the AMR technique with the multidirect forcing immersed boundary method to resolve particles only on the finest-level grid. It significantly reduces the grid requirements for particle-resolved simulation compared with commonly used uniform grid solvers [Incompact3d](https://github.com/xcompact3d/Incompact3d), [CaNS](https://github.com/CaNS-World/CaNS), and [CP3d](https://github.com/GongZheng-Justin/CP3d). Additionally, we utilized a subcycling technique to alleviate the time step constraint on coarser levels. It minimizes the total time step needed by time advancement compared with the non-subcycling technique used in other AMR-related packages, such as [IBAMR](https://github.com/IBAMR/IBAMR.git), [basilisk](http://basilisk.fr/), and [incflo](https://github.com/AMReX-Fluids/incflo.git).
+
+## Get Help
+
+You can also view questions
+and ask your own on our [GitHub Discussions](https://github.com/ruohai0925/IAMReX/discussions) page.
+To obtain additional help, simply post an issue.
+
+## Contribute
+
+We are always happy to have users contribute to the IAMReX source code. To
+contribute, issue a pull request against the development branch.
+Any level of changes are welcomed: documentation, bug fixes, new test problems,
+new solvers, etc. For more details on how to contribute to IAMReX, please see
+[CONTRIBUTING.md](CONTRIBUTING.md).
+
+💡 If you're using IAMReX in your own GitHub projects, consider adding `IAMReX`
+as a [repository topic](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/classifying-your-repository-with-topics)!
+This helps others discover related work and strengthens the IAMReX ecosystem.
+
+## Citation
+
+To cite IAMReX, please use
+
+```
+@article{10.1063/5.0236509,
+ author = {Li, Xuzhu (李虚竹) and Li, Chun (李春) and Li, Xiaokai (李晓凯) and Li, Wenzhuo (李文卓) and Tang, Mingze (唐铭泽) and Zeng, Yadong (曾亚东) and Zhu, Zhengping (朱正平)},
+ title = {An open-source, adaptive solver for particle-resolved simulations with both subcycling and non-subcycling methods},
+ journal = {Physics of Fluids},
+ volume = {36},
+ number = {11},
+ pages = {113335},
+ year = {2024},
+ month = {11},
+ issn = {1070-6631},
+ doi = {10.1063/5.0236509},
+ url = {https://doi.org/10.1063/5.0236509},
+ eprint = {https://pubs.aip.org/aip/pof/article-pdf/doi/10.1063/5.0236509/20247663/113335\_1\_5.0236509.pdf},
+}
+
+@inbook{doi:10.2514/6.2025-1865,
+ author = {Dewen Liu and Shuai He and Haoran Cheng and Yadong Zeng},
+ title = {Investigate the Efficiency of Incompressible Flow Simulations on CPUs and GPUs With BSAMR},
+ booktitle = {AIAA SCITECH 2025 Forum},
+ doi = {10.2514/6.2025-1865},
+ URL = {https://arc.aiaa.org/doi/abs/10.2514/6.2025-1865},
+ eprint = {https://arc.aiaa.org/doi/pdf/10.2514/6.2025-1865},
+}
+
+```
+
+## Acknowledgements
+
+We are grateful to Ann Almgren, Andy Nonaka, Andrew Myers, Axel Huebl, Marc Day, and Weiqun Zhang in the Lawrence Berkeley National Laboratory (LBNL) for their discussions related to [AMReX](https://github.com/AMReX-Codes/amrex) and [IAMR](https://github.com/AMReX-Fluids/IAMR). Y.Z. and Z.Z. also thank Prof. Lian Shen, Prof. Ruifeng Hu, and Prof. Xiaojing Zheng during their Ph.D. studies.
+
+
+
+## Contact
+
+If you have any question or wanna contribute to the code, please don't hesitate to contact us via the [GitHub Issues](https://github.com/ruohai0925/IAMReX/issues) or zdsjtu@gmail.com.
diff --git a/Source/Collision.H b/Source/Collision.H
index 802606af..8bc9ce29 100644
--- a/Source/Collision.H
+++ b/Source/Collision.H
@@ -1,3 +1,9 @@
+/*
+ * SPDX-FileCopyrightText: 2023 - 2025 Yadong Zeng & ZhuXu Li<1246206018@qq.com>
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
#ifndef COLLISION_H
#define COLLISION_H
diff --git a/Source/Collision.cpp b/Source/Collision.cpp
index 576be555..c2305287 100644
--- a/Source/Collision.cpp
+++ b/Source/Collision.cpp
@@ -1,3 +1,7 @@
+// SPDX-FileCopyrightText: 2023 - 2025 Yadong Zeng & ZhuXu Li<1246206018@qq.com>
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
#include "Collision.H"
#include
diff --git a/Source/DiffusedIB.H b/Source/DiffusedIB.H
index 3cc7da8a..445e1568 100644
--- a/Source/DiffusedIB.H
+++ b/Source/DiffusedIB.H
@@ -1,201 +1,207 @@
-#ifndef DIFFUSEDIB_H
-#define DIFFUSEDIB_H
-
-#include
-#include
-
-#include
-#include
-
-#include "Collision.H"
-using namespace amrex;
-// using deltaFuncType = std::function;
-
-AMREX_INLINE AMREX_GPU_DEVICE
-Real nodal_phi_to_heavi(Real phi)
-{
- if (phi <= 0.0) {
- return 0.0;
- }
- else {
- return 1.0;
- }
-}
-
-void nodal_phi_to_pvf(MultiFab& pvf, const MultiFab& phi_nodal);
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* particle and markers */
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-enum P_ATTR{
- U_Marker = 0,
- V_Marker,
- W_Marker,
- Fx_Marker,
- Fy_Marker,
- Fz_Marker,
- Mx_Marker,
- My_Marker,
- Mz_Marker,
- numAttri
-};
-
-enum DELTA_FUNCTION_TYPE{
- FOUR_POINT_IB = 0,
- THREE_POINT_IB
-};
-
-struct kernel{
- int id;
- RealVect velocity{0.0,0.0,0.0};
- RealVect location{0.0,0.0,0.0};
- RealVect omega{0.0,0.0,0.0};
-
- RealVect velocity_old{0.0,0.0,0.0};
- RealVect location_old{0.0,0.0,0.0};
- RealVect omega_old{0.0,0.0,0.0};
-
- RealVect varphi{0.0,0.0,0.0};
- Real radius{0.0};
- Real rho{0.0};
- int ml{0};
- Real dv{0.0};
- Real Vp{0.0};
- RealVect ib_force{0.0,0.0,0.0};
- RealVect ib_moment{0.0, 0.0, 0.0};
-
- RealVect sum_u_new{0.0,0.0,0.0};
- RealVect sum_u_old{0.0,0.0,0.0};
- RealVect sum_t_new{0.0,0.0,0.0};
- RealVect sum_t_old{0.0,0.0,0.0};
-
- RealVect Fcp{0.0,0.0,0.0};
- RealVect Tcp{0.0,0.0,0.0};
-
- amrex::Gpu::DeviceVector phiK;
- amrex::Gpu::DeviceVector thetaK;
-
- IntVect TL{0, 0, 0}, RL{0, 0, 0};
-};
-
-class mParIter : public ParIter<0, 0, numAttri>{
-public:
- using amrex::ParIter<0, 0, numAttri>::ParIter;
- using RealVector = amrex::ParIter<0, 0, numAttri>::ContainerType::RealVector;
-
- [[nodiscard]] const std::array& GetAttribs () const {
- return GetStructOfArrays().GetRealData();
- }
-
- [[nodiscard]] const RealVector& GetAttribs (int comp) const {
- return GetStructOfArrays().GetRealData(comp);
- }
-
- std::array& GetAttribs () {
- return GetStructOfArrays().GetRealData();
- }
-
- RealVector& GetAttribs (int comp) {
- return GetStructOfArrays().GetRealData(comp);
- }
-};
-
-void deltaFunction(Real xf, Real xp, Real h, Real& value);
-
-using mParticleContainer = ParticleContainer<0, 0, numAttri>;
-
-class mParticle
-{
-public:
- explicit mParticle() = default;
-
- void InitParticles(const Vector& x,
- const Vector& y,
- const Vector& z,
- const Vector& rho_s,
- const Vector& Vx,
- const Vector& Vy,
- const Vector& Vz,
- const Vector& Ox,
- const Vector& Oy,
- const Vector& Oz,
- const Vector& TLX,
- const Vector& TLY,
- const Vector& TLZ,
- const Vector& RLX,
- const Vector& RLY,
- const Vector& RLZ,
- const Vector& radius,
- Real h,
- Real gravity,
- int verbose = 0);
-
- void InteractWithEuler(MultiFab &EulerVel, MultiFab &EulerForce, Real dt = 0.1, DELTA_FUNCTION_TYPE type = FOUR_POINT_IB);
-
- void WriteParticleFile(int index);
-
- void InitialWithLargrangianPoints(const kernel& current_kernel);
-
- void ResetLargrangianPoints();
-
- void VelocityInterpolation(amrex::MultiFab &Euler, DELTA_FUNCTION_TYPE type);
-
- void ComputeLagrangianForce(Real dt, const kernel& kernel);
-
- void ForceSpreading(amrex::MultiFab &Euler, kernel& kernel, DELTA_FUNCTION_TYPE type);
-
- void VelocityCorrection(amrex::MultiFab &Euler, amrex::MultiFab &EulerForce, Real dt) const;
-
- void UpdateParticles(int iStep, Real time, const MultiFab& Euler_old, const MultiFab& Euler, MultiFab& phi_nodal, MultiFab& pvf, Real dt);
-
- void DoParticleCollision(int model);
-
- static void WriteIBForceAndMoment(int step, amrex::Real time, amrex::Real dt, kernel& current_kernel);
-
- void RecordOldValue(kernel& kernel);
-
- Vector particle_kernels;
-
- mParticleContainer *mContainer{nullptr};
-
- ParticleCollision m_Collision;
-
- int max_largrangian_num = 0;
-
- uint32_t ib_force_file_index = 0;
-
- RealVect m_gravity{0.0,0.0,0.0};
-
- int verbose = 0;
-
- Real spend_time;
-};
-
-class Particles{
-public:
- static void create_particles(const Geometry &gm,
- const DistributionMapping & dm,
- const BoxArray & ba);
-
- static mParticle* get_particles();
- static void init_particle(Real gravity, Real h);
- static void Restart(Real gravity, Real h, int iStep);
- static void Initialize();
- static int ParticleFinestLevel();
- static void define_para(const Vector& x,
- const Vector& y,
- const Vector& z,
- Real rho_s,
- Real radius,
- Real rho_f,
- int force_index,
- int velocity_index,
- int finest_level);
-
- inline static bool isInitial{false};
-private:
- inline static mParticle* particle = nullptr;
-};
-
-#endif
+/*
+ * SPDX-FileCopyrightText: 2023 - 2025 Yadong Zeng & ZhuXu Li<1246206018@qq.com>
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef DIFFUSEDIB_H
+#define DIFFUSEDIB_H
+
+#include
+#include
+
+#include
+#include
+
+#include "Collision.H"
+using namespace amrex;
+// using deltaFuncType = std::function;
+
+AMREX_INLINE AMREX_GPU_DEVICE
+Real nodal_phi_to_heavi(Real phi)
+{
+ if (phi <= 0.0) {
+ return 0.0;
+ }
+ else {
+ return 1.0;
+ }
+}
+
+void nodal_phi_to_pvf(MultiFab& pvf, const MultiFab& phi_nodal);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* particle and markers */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+enum P_ATTR{
+ U_Marker = 0,
+ V_Marker,
+ W_Marker,
+ Fx_Marker,
+ Fy_Marker,
+ Fz_Marker,
+ Mx_Marker,
+ My_Marker,
+ Mz_Marker,
+ numAttri
+};
+
+enum DELTA_FUNCTION_TYPE{
+ FOUR_POINT_IB = 0,
+ THREE_POINT_IB
+};
+
+struct kernel{
+ int id;
+ RealVect velocity{0.0,0.0,0.0};
+ RealVect location{0.0,0.0,0.0};
+ RealVect omega{0.0,0.0,0.0};
+
+ RealVect velocity_old{0.0,0.0,0.0};
+ RealVect location_old{0.0,0.0,0.0};
+ RealVect omega_old{0.0,0.0,0.0};
+
+ RealVect varphi{0.0,0.0,0.0};
+ Real radius{0.0};
+ Real rho{0.0};
+ int ml{0};
+ Real dv{0.0};
+ Real Vp{0.0};
+ RealVect ib_force{0.0,0.0,0.0};
+ RealVect ib_moment{0.0, 0.0, 0.0};
+
+ RealVect sum_u_new{0.0,0.0,0.0};
+ RealVect sum_u_old{0.0,0.0,0.0};
+ RealVect sum_t_new{0.0,0.0,0.0};
+ RealVect sum_t_old{0.0,0.0,0.0};
+
+ RealVect Fcp{0.0,0.0,0.0};
+ RealVect Tcp{0.0,0.0,0.0};
+
+ amrex::Gpu::DeviceVector phiK;
+ amrex::Gpu::DeviceVector thetaK;
+
+ IntVect TL{0, 0, 0}, RL{0, 0, 0};
+};
+
+class mParIter : public ParIter<0, 0, numAttri>{
+public:
+ using amrex::ParIter<0, 0, numAttri>::ParIter;
+ using RealVector = amrex::ParIter<0, 0, numAttri>::ContainerType::RealVector;
+
+ [[nodiscard]] const std::array& GetAttribs () const {
+ return GetStructOfArrays().GetRealData();
+ }
+
+ [[nodiscard]] const RealVector& GetAttribs (int comp) const {
+ return GetStructOfArrays().GetRealData(comp);
+ }
+
+ std::array& GetAttribs () {
+ return GetStructOfArrays().GetRealData();
+ }
+
+ RealVector& GetAttribs (int comp) {
+ return GetStructOfArrays().GetRealData(comp);
+ }
+};
+
+void deltaFunction(Real xf, Real xp, Real h, Real& value);
+
+using mParticleContainer = ParticleContainer<0, 0, numAttri>;
+
+class mParticle
+{
+public:
+ explicit mParticle() = default;
+
+ void InitParticles(const Vector& x,
+ const Vector& y,
+ const Vector& z,
+ const Vector& rho_s,
+ const Vector& Vx,
+ const Vector& Vy,
+ const Vector& Vz,
+ const Vector& Ox,
+ const Vector& Oy,
+ const Vector& Oz,
+ const Vector& TLX,
+ const Vector& TLY,
+ const Vector& TLZ,
+ const Vector& RLX,
+ const Vector& RLY,
+ const Vector& RLZ,
+ const Vector& radius,
+ Real h,
+ Real gravity,
+ int verbose = 0);
+
+ void InteractWithEuler(MultiFab &EulerVel, MultiFab &EulerForce, Real dt = 0.1, DELTA_FUNCTION_TYPE type = FOUR_POINT_IB);
+
+ void WriteParticleFile(int index);
+
+ void InitialWithLargrangianPoints(const kernel& current_kernel);
+
+ void ResetLargrangianPoints();
+
+ void VelocityInterpolation(amrex::MultiFab &Euler, DELTA_FUNCTION_TYPE type);
+
+ void ComputeLagrangianForce(Real dt, const kernel& kernel);
+
+ void ForceSpreading(amrex::MultiFab &Euler, kernel& kernel, DELTA_FUNCTION_TYPE type);
+
+ void VelocityCorrection(amrex::MultiFab &Euler, amrex::MultiFab &EulerForce, Real dt) const;
+
+ void UpdateParticles(int iStep, Real time, const MultiFab& Euler_old, const MultiFab& Euler, MultiFab& phi_nodal, MultiFab& pvf, Real dt);
+
+ void DoParticleCollision(int model);
+
+ static void WriteIBForceAndMoment(int step, amrex::Real time, amrex::Real dt, kernel& current_kernel);
+
+ void RecordOldValue(kernel& kernel);
+
+ Vector particle_kernels;
+
+ mParticleContainer *mContainer{nullptr};
+
+ ParticleCollision m_Collision;
+
+ int max_largrangian_num = 0;
+
+ uint32_t ib_force_file_index = 0;
+
+ RealVect m_gravity{0.0,0.0,0.0};
+
+ int verbose = 0;
+
+ Real spend_time;
+};
+
+class Particles{
+public:
+ static void create_particles(const Geometry &gm,
+ const DistributionMapping & dm,
+ const BoxArray & ba);
+
+ static mParticle* get_particles();
+ static void init_particle(Real gravity, Real h);
+ static void Restart(Real gravity, Real h, int iStep);
+ static void Initialize();
+ static int ParticleFinestLevel();
+ static void define_para(const Vector& x,
+ const Vector& y,
+ const Vector& z,
+ Real rho_s,
+ Real radius,
+ Real rho_f,
+ int force_index,
+ int velocity_index,
+ int finest_level);
+
+ inline static bool isInitial{false};
+private:
+ inline static mParticle* particle = nullptr;
+};
+
+#endif
diff --git a/Source/DiffusedIB.cpp b/Source/DiffusedIB.cpp
index 704a1bea..2dc473f9 100644
--- a/Source/DiffusedIB.cpp
+++ b/Source/DiffusedIB.cpp
@@ -1,1222 +1,1225 @@
-
-#include
-#include
-#include
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include
-#include
-namespace fs = std::filesystem;
-
-#define GHOST_CELLS 2
-
-using namespace amrex;
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* global variable */
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#define LOCAL_LEVEL 0
-
-const Vector direction_str{"X","Y","Z"};
-
-namespace ParticleProperties{
- Vector _x{}, _y{}, _z{}, _rho{};
- Vector Vx{}, Vy{}, Vz{};
- Vector Ox{}, Oy{}, Oz{};
- Vector _radius;
- Real rd{0.0};
- Vector TLX{}, TLY{},TLZ{},RLX{},RLY{},RLZ{};
- int euler_finest_level{0};
- int euler_velocity_index{0};
- int euler_force_index{0};
- Real euler_fluid_rho{0.0};
- int verbose{0};
- int loop_ns{2};
- int loop_solid{1};
- int Uhlmann{0};
-
- Vector GLO, GHI;
- int start_step{-1};
- int collision_model{0};
-
- int write_freq{1};
- bool init_particle_from_file{false};
-
- GpuArray plo{0.0,0.0,0.0}, phi{0.0,0.0,0.0}, dx{0.0, 0.0, 0.0};
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* other function */
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-void nodal_phi_to_pvf(MultiFab& pvf, const MultiFab& phi_nodal)
-{
-
- // amrex::Print() << "In the nodal_phi_to_pvf\n";
-
- pvf.setVal(0.0);
-
- // Only set the valid cells of pvf
-#ifdef AMREX_USE_OMP
-#pragma omp parallel if (Gpu::notInLaunchRegion())
-#endif
- for (MFIter mfi(pvf,TilingIfNotGPU()); mfi.isValid(); ++mfi)
- {
- const Box& bx = mfi.tilebox();
- auto const& pvffab = pvf.array(mfi);
- auto const& pnfab = phi_nodal.array(mfi);
- amrex::ParallelFor(bx, [pvffab, pnfab]
- AMREX_GPU_DEVICE(int i, int j, int k) noexcept
- {
- Real num = 0.0;
- for(int kk=k; kk<=k+1; kk++) {
- for(int jj=j; jj<=j+1; jj++) {
- for(int ii=i; ii<=i+1; ii++) {
- num += (-pnfab(ii,jj,kk)) * nodal_phi_to_heavi(-pnfab(ii,jj,kk));
- }
- }
- }
- Real deo = 0.0;
- for(int kk=k; kk<=k+1; kk++) {
- for(int jj=j; jj<=j+1; jj++) {
- for(int ii=i; ii<=i+1; ii++) {
- deo += std::abs(pnfab(ii,jj,kk));
- }
- }
- }
- pvffab(i,j,k) = num / (deo + 1.e-12);
- });
- }
-
-}
-
-void calculate_phi_nodal(MultiFab& phi_nodal, kernel& current_kernel)
-{
- phi_nodal.setVal(0.0);
-
- amrex::Real Xp = current_kernel.location[0];
- amrex::Real Yp = current_kernel.location[1];
- amrex::Real Zp = current_kernel.location[2];
- amrex::Real Rp = current_kernel.radius;
-
- // Only set the valid cells of phi_nodal
- for (MFIter mfi(phi_nodal,TilingIfNotGPU()); mfi.isValid(); ++mfi)
- {
- const Box& bx = mfi.tilebox();
- auto const& pnfab = phi_nodal.array(mfi);
- auto dx = ParticleProperties::dx;
- auto plo = ParticleProperties::plo;
- amrex::ParallelFor(bx, [=]
- AMREX_GPU_DEVICE(int i, int j, int k) noexcept
- {
- Real Xn = i * dx[0] + plo[0];
- Real Yn = j * dx[1] + plo[1];
- Real Zn = k * dx[2] + plo[2];
-
- pnfab(i,j,k) = std::sqrt( (Xn - Xp)*(Xn - Xp)
- + (Yn - Yp)*(Yn - Yp) + (Zn - Zp)*(Zn - Zp)) - Rp;
- pnfab(i,j,k) = pnfab(i,j,k) / Rp;
-
- }
- );
- }
-}
-
-// May use ParReduce later, https://amrex-codes.github.io/amrex/docs_html/GPU.html#multifab-reductions
-void CalculateSumU_cir (RealVect& sum,
- const MultiFab& E,
- const MultiFab& pvf,
- int EulerVelIndex)
-{
- auto const& E_data = E.const_arrays();
- auto const& pvf_data = pvf.const_arrays();
- const Real d = Math::powi<3>(ParticleProperties::dx[0]);
- amrex::GpuTuple tmpSum = ParReduce(TypeList{}, TypeList{},E, IntVect{0},
- [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept -> amrex::GpuTuple{
- auto E_ = E_data[box_no];
- auto pvf_ = pvf_data[box_no];
- return {
- E_(i, j, k, EulerVelIndex ) * d * pvf_(i,j,k),
- E_(i, j, k, EulerVelIndex + 1) * d * pvf_(i,j,k),
- E_(i, j, k, EulerVelIndex + 2) * d * pvf_(i,j,k)
- };
- });
- sum[0] = amrex::get<0>(tmpSum);
- sum[1] = amrex::get<1>(tmpSum);
- sum[2] = amrex::get<2>(tmpSum);
-}
-
-void CalculateSumT_cir (RealVect& sum,
- const MultiFab& E,
- const MultiFab& pvf,
- const RealVect pLoc,
- int EulerVelIndex)
-{
- auto plo = ParticleProperties::plo;
- auto dx = ParticleProperties::dx;
-
- auto const& E_data = E.const_arrays();
- auto const& pvf_data = pvf.const_arrays();
- const Real d = Math::powi<3>(ParticleProperties::dx[0]);
- amrex::GpuTuple tmpSum = ParReduce(TypeList{}, TypeList{},E, IntVect{0},
- [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept -> amrex::GpuTuple{
- auto E_ = E_data[box_no];
- auto pvf_ = pvf_data[box_no];
-
- Real x = plo[0] + i*dx[0] + 0.5*dx[0];
- Real y = plo[1] + j*dx[1] + 0.5*dx[1];
- Real z = plo[2] + k*dx[2] + 0.5*dx[2];
-
- Real vx = E_(i, j, k, EulerVelIndex );
- Real vy = E_(i, j, k, EulerVelIndex + 1);
- Real vz = E_(i, j, k, EulerVelIndex + 2);
-
- RealVect tmp = RealVect(x - pLoc[0], y - pLoc[1], z - pLoc[2]).crossProduct(RealVect(vx, vy, vz));
-
- return {
- tmp[0] * d * pvf_(i, j, k),
- tmp[1] * d * pvf_(i, j, k),
- tmp[2] * d * pvf_(i, j, k)
- };
- });
- sum[0] = amrex::get<0>(tmpSum);
- sum[1] = amrex::get<1>(tmpSum);
- sum[2] = amrex::get<2>(tmpSum);
-}
-
-[[nodiscard]] AMREX_FORCE_INLINE
-Real cal_momentum(Real rho, Real radius)
-{
- return 8.0 * Math::pi() * rho * Math::powi<5>(radius) / 15.0;
-}
-
-AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-void deltaFunction(Real xf, Real xp, Real h, Real& value, DELTA_FUNCTION_TYPE type)
-{
- Real rr = amrex::Math::abs(( xf - xp ) / h);
-
- switch (type) {
- case DELTA_FUNCTION_TYPE::FOUR_POINT_IB:
- if(rr >= 0 && rr < 1 ){
- value = 1.0 / 8.0 * ( 3.0 - 2.0 * rr + std::sqrt( 1.0 + 4.0 * rr - 4 * Math::powi<2>(rr))) / h;
- }else if (rr >= 1 && rr < 2) {
- value = 1.0 / 8.0 * ( 5.0 - 2.0 * rr - std::sqrt( -7.0 + 12.0 * rr - 4 * Math::powi<2>(rr))) / h;
- }else {
- value = 0;
- }
- break;
- case DELTA_FUNCTION_TYPE::THREE_POINT_IB:
- if(rr >= 0.5 && rr < 1.5){
- value = 1.0 / 6.0 * ( 5.0 - 3.0 * rr - std::sqrt( - 3.0 * Math::powi<2>( 1 - rr) + 1.0 )) / h;
- }else if (rr >= 0 && rr < 0.5) {
- value = 1.0 / 3.0 * ( 1.0 + std::sqrt( 1.0 - 3 * Math::powi<2>(rr))) / h;
- }else {
- value = 0;
- }
- break;
- default:
- break;
- }
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* mParticle member function */
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-//loop all particels
-void mParticle::InteractWithEuler(MultiFab &EulerVel,
- MultiFab &EulerForce,
- Real dt,
- DELTA_FUNCTION_TYPE type)
-{
- if (verbose) amrex::Print() << "[Particle] mParticle::InteractWithEuler\n";
- // clear time , start record
- spend_time = 0;
- auto InteractWithEulerStart = ParallelDescriptor::second();
-
- MultiFab EulerForceTmp(EulerForce.boxArray(), EulerForce.DistributionMap(), 3, EulerForce.nGrow());
- //clean preStep's IB_porperties
- for(auto& kernel : particle_kernels) {
- kernel.ib_force.scale(0.0);
- kernel.ib_moment.scale(0.0);
- }
-
- //for 1 -> Ns
- int loop = ParticleProperties::loop_ns;
- BL_ASSERT(loop > 0);
- while(loop > 0){
- if(verbose) amrex::Print() << "[Particle] Ns loop index : " << loop << "\n";
-
- EulerForce.setVal(0.0);
-
- for(kernel& kernel : particle_kernels){
- InitialWithLargrangianPoints(kernel); // Initialize markers for a specific particle
- ResetLargrangianPoints();
- EulerForceTmp.setVal(0.0);
- auto ib_force = kernel.ib_force;
- auto ib_moment = kernel.ib_moment;
- kernel.ib_force.scale(0.0); // clear kernel ib_force
- kernel.ib_moment.scale(0.0); // clear kernel ib_moment
-
- VelocityInterpolation(EulerVel, type);
- ComputeLagrangianForce(dt, kernel);
- ForceSpreading(EulerForceTmp, kernel, type);
- MultiFab::Add(EulerForce, EulerForceTmp, 0, 0, 3, EulerForce.nGrow());
-
- kernel.ib_force += ib_force;
- kernel.ib_moment += ib_moment;
- }
- VelocityCorrection(EulerVel, EulerForce, dt);
- loop--;
- }
- spend_time += ParallelDescriptor::second() - InteractWithEulerStart;
-}
-
-void mParticle::InitParticles(const Vector& x,
- const Vector& y,
- const Vector& z,
- const Vector& rho_s,
- const Vector& Vx,
- const Vector& Vy,
- const Vector& Vz,
- const Vector& Ox,
- const Vector& Oy,
- const Vector& Oz,
- const Vector& TLXt,
- const Vector& TLYt,
- const Vector& TLZt,
- const Vector& RLXt,
- const Vector& RLYt,
- const Vector& RLZt,
- const Vector& radius,
- Real h,
- Real gravity,
- int _verbose)
-{
- verbose = _verbose;
- if (verbose) amrex::Print() << "[Particle] mParticle::InitParticles\n";
-
- m_gravity[2] = gravity;
-
- //pre judge
- if(!((x.size() == y.size()) && (x.size() == z.size()))){
- Print() << "particle's position container are all different size";
- return;
- }
-
- //all the particles have different radius
- for(int index = 0; index < x.size(); index++){
- int real_index;
- // if initial with input file, initialized by [0] data
- if(ParticleProperties::init_particle_from_file){
- real_index = 0;
- }else{
- real_index = index;
- }
-
- kernel mKernel;
- mKernel.id = index + 1;
- mKernel.location[0] = x[index];
- mKernel.location[1] = y[index];
- mKernel.location[2] = z[index];
- mKernel.velocity[0] = Vx[real_index];
- mKernel.velocity[1] = Vy[real_index];
- mKernel.velocity[2] = Vz[real_index];
- mKernel.omega[0] = Ox[real_index];
- mKernel.omega[1] = Oy[real_index];
- mKernel.omega[2] = Oz[real_index];
-
- // use current state to initialize old state
- mKernel.location_old = mKernel.location;
- mKernel.velocity_old = mKernel.velocity;
- mKernel.omega_old = mKernel.omega;
-
- mKernel.TL[0] = TLXt[real_index];
- mKernel.TL[1] = TLYt[real_index];
- mKernel.TL[2] = TLZt[real_index];
- mKernel.RL[0] = RLXt[real_index];
- mKernel.RL[1] = RLYt[real_index];
- mKernel.RL[2] = RLZt[real_index];
- mKernel.rho = rho_s[real_index];
- mKernel.radius = radius[real_index];
- mKernel.Vp = Math::pi() * 4 / 3 * Math::powi<3>(radius[real_index]);
-
- //int Ml = static_cast( Math::pi() / 3 * (12 * Math::powi<2>(mKernel.radius / h)));
- //Real dv = Math::pi() * h / 3 / Ml * (12 * mKernel.radius * mKernel.radius + h * h);
- int Ml = static_cast((amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd - 0.5) * h)
- - amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd + 0.5) * h))/(3.*h*h*h/4./Math::pi()));
- Real dv = (amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd - 0.5) * h)
- - amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd + 0.5) * h))/(3.*Ml/4./Math::pi());
- mKernel.ml = Ml;
- mKernel.dv = dv;
- if( Ml > max_largrangian_num ) max_largrangian_num = Ml;
-
- Real phiK = 0;
- for(int marker_index = 0; marker_index < Ml; marker_index++){
- Real Hk = -1.0 + 2.0 * (marker_index) / ( Ml - 1.0);
- Real thetaK = std::acos(Hk);
- if(marker_index == 0 || marker_index == (Ml - 1)){
- phiK = 0;
- }else {
- phiK = std::fmod( phiK + 3.809 / std::sqrt(Ml) / std::sqrt( 1 - Math::powi<2>(Hk)) , 2 * Math::pi());
- }
- mKernel.phiK.push_back(phiK);
- mKernel.thetaK.push_back(thetaK);
- }
-
- particle_kernels.emplace_back(mKernel);
-
- if (verbose) amrex::Print() << "h: " << h << ", Ml: " << Ml << ", D: " << Math::powi<3>(h) << " gravity : " << gravity << "\n"
- << "Kernel : " << index << ": Location (" << x[index] << ", " << y[index] << ", " << z[index]
- << "), Velocity : (" << mKernel.velocity[0] << ", " << mKernel.velocity[1] << ", "<< mKernel.velocity[2]
- << "), Radius: " << mKernel.radius << ", Ml: " << Ml << ", dv: " << dv << ", Rho: " << mKernel.rho << "\n";
- }
- //collision box generate
- m_Collision.SetGeometry(RealVect(ParticleProperties::GLO), RealVect(ParticleProperties::GHI),particle_kernels[0].radius, h);
-}
-
-void mParticle::InitialWithLargrangianPoints(const kernel& current_kernel){
-
- if (verbose) amrex::Print() << "mParticle::InitialWithLargrangianPoints\n";
- for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
- const Long np = pti.numParticles();
- if(np == 0) continue;
- auto *particles = pti.GetArrayOfStructs().data();
-
- const auto location = current_kernel.location;
- const auto radius = current_kernel.radius;
- const auto* phiK = current_kernel.phiK.dataPtr();
- const auto* thetaK = current_kernel.thetaK.dataPtr();
-
- amrex::ParallelFor( np, [=]
- AMREX_GPU_DEVICE (int i) noexcept {
- auto id = particles[i].id();
- particles[i].pos(0) = location[0] + radius * std::sin(thetaK[id - 1]) * std::cos(phiK[id - 1]);
- particles[i].pos(1) = location[1] + radius * std::sin(thetaK[id - 1]) * std::sin(phiK[id - 1]);
- particles[i].pos(2) = location[2] + radius * std::cos(thetaK[id - 1]);
- }
- );
- }
- // Redistribute the markers after updating their locations
- mContainer->Redistribute();
- if (verbose) {
- amrex::Print() << "[particle] : particle num :" << mContainer->TotalNumberOfParticles() << "\n";
- mContainer->WriteAsciiFile(amrex::Concatenate("particle", 1));
- }
-}
-
-template >
-AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-void VelocityInterpolation_cir(int p_iter, P const& p, Real& Up, Real& Vp, Real& Wp,
- Array4 const& E, int EulerVIndex,
- const int *lo, const int *hi,
- GpuArray const& plo,
- GpuArray const& dx,
- DELTA_FUNCTION_TYPE type)
-{
-
- //std::cout << "lo " << lo[0] << " " << lo[1] << " "<< lo[2] << "\n";
- //std::cout << "hi " << hi[0] << " " << hi[1] << " "<< hi[2] << "\n";
-
- const Real d = AMREX_D_TERM(dx[0], *dx[1], *dx[2]);
-
- const Real lx = (p.pos(0) - plo[0]) / dx[0]; // x
- const Real ly = (p.pos(1) - plo[1]) / dx[1]; // y
- const Real lz = (p.pos(2) - plo[2]) / dx[2]; // z
-
- int i = static_cast(Math::floor(lx)); // i
- int j = static_cast(Math::floor(ly)); // j
- int k = static_cast(Math::floor(lz)); // k
-
- //std::cout << "p_iter " << p_iter << " p.pos(0): " << p.pos(0) << " p.pos(1): " << p.pos(1) << " p.pos(2): " << p.pos(2) << "\n";
-
- // std::cout << "d: " << d << "\n"
- // << "lx: " << lx << ", ly: " << ly << ", lz: " << lz << "\n"
- // << "i: " << i << ", j: " << j << ", k: " << k << std::endl;
-
- Up = 0;
- Vp = 0;
- Wp = 0;
- //Euler to largrangian
- for(int ii = -2; ii < 3; ii++){
- for(int jj = -2; jj < 3; jj++){
- for(int kk = -2; kk < 3; kk ++){
- Real tU, tV, tW;
- const Real xi = plo[0] + (i + ii) * dx[0] + dx[0]/2;
- const Real yj = plo[1] + (j + jj) * dx[1] + dx[1]/2;
- const Real kz = plo[2] + (k + kk) * dx[2] + dx[2]/2;
- deltaFunction( p.pos(0), xi, dx[0], tU, type);
- deltaFunction( p.pos(1), yj, dx[1], tV, type);
- deltaFunction( p.pos(2), kz, dx[2], tW, type);
- const Real delta_value = tU * tV * tW;
- Up += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex ) * d;
- Vp += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex + 1) * d;
- Wp += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex + 2) * d;
- }
- }
- }
-}
-
-void mParticle::VelocityInterpolation(MultiFab &EulerVel,
- DELTA_FUNCTION_TYPE type)//
-{
- if (verbose) amrex::Print() << "\tmParticle::VelocityInterpolation\n";
-
- //amrex::Print() << "euler_finest_level " << euler_finest_level << std::endl;
- const auto& gm = mContainer->GetParGDB()->Geom(LOCAL_LEVEL);
- auto plo = gm.ProbLoArray();
- auto dx = gm.CellSizeArray();
- // attention
- // velocity ghost cells will be up-to-date
- EulerVel.FillBoundary(ParticleProperties::euler_velocity_index, 3, gm.periodicity());
-
- const int EulerVelocityIndex = ParticleProperties::euler_velocity_index;
-
- for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
-
- const Box& box = pti.validbox();
-
- auto& particles = pti.GetArrayOfStructs();
- auto *p_ptr = particles.data();
- const Long np = pti.numParticles();
-
- auto& attri = pti.GetAttribs();
- auto* Up = attri[P_ATTR::U_Marker].data();
- auto* Vp = attri[P_ATTR::V_Marker].data();
- auto* Wp = attri[P_ATTR::W_Marker].data();
- const auto& E = EulerVel.array(pti);
-
- amrex::ParallelFor(np, [=]
- AMREX_GPU_DEVICE (int i) noexcept{
- VelocityInterpolation_cir(i, p_ptr[i], Up[i], Vp[i], Wp[i], E, EulerVelocityIndex, box.loVect(), box.hiVect(), plo, dx, type);
- });
- }
- if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 2));
- //amrex::Abort("stop here!");
-}
-
-template
-AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-void ForceSpreading_cic (P const& p,
- Real Px,
- Real Py,
- Real Pz,
- ParticleReal& fxP,
- ParticleReal& fyP,
- ParticleReal& fzP,
- ParticleReal& mxP,
- ParticleReal& myP,
- ParticleReal& mzP,
- Array4 const& E,
- int EulerForceIndex,
- Real dv,
- GpuArray const& plo,
- GpuArray const& dx,
- DELTA_FUNCTION_TYPE type)
-{
- //const Real d = AMREX_D_TERM(dx[0], *dx[1], *dx[2]);
- //plo to ii jj kk
- Real lx = (p.pos(0) - plo[0]) / dx[0];
- Real ly = (p.pos(1) - plo[1]) / dx[1];
- Real lz = (p.pos(2) - plo[2]) / dx[2];
-
- int i = static_cast(Math::floor(lx));
- int j = static_cast(Math::floor(ly));
- int k = static_cast(Math::floor(lz));
- fxP *= dv;
- fyP *= dv;
- fzP *= dv;
- RealVect moment = RealVect((p.pos(0) - Px), (p.pos(1) - Py), (p.pos(2) - Pz)).crossProduct(RealVect(fxP, fyP, fzP));
- mxP = moment[0];
- myP = moment[1];
- mzP = moment[2];
- //lagrangian to Euler
- for(int ii = -2; ii < +3; ii++){
- for(int jj = -2; jj < +3; jj++){
- for(int kk = -2; kk < +3; kk ++){
- Real tU, tV, tW;
- const Real xi =plo[0] + (i + ii) * dx[0] + dx[0]/2;
- const Real yj =plo[1] + (j + jj) * dx[1] + dx[1]/2;
- const Real kz =plo[2] + (k + kk) * dx[2] + dx[2]/2;
- deltaFunction( p.pos(0), xi, dx[0], tU, type);
- deltaFunction( p.pos(1), yj, dx[1], tV, type);
- deltaFunction( p.pos(2), kz, dx[2], tW, type);
- Real delta_value = tU * tV * tW;
- Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex ), delta_value * fxP);
- Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex+1), delta_value * fyP);
- Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex+2), delta_value * fzP);
- }
- }
- }
-}
-
-void mParticle::ForceSpreading(MultiFab & EulerForce,
- kernel& kernel,
- DELTA_FUNCTION_TYPE type)
-{
- if (verbose) amrex::Print() << "\tmParticle::ForceSpreading\n";
- const auto& gm = mContainer->GetParGDB()->Geom(LOCAL_LEVEL);
- auto plo = gm.ProbLoArray();
- auto dxi = gm.CellSizeArray();
- for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
- const Long np = pti.numParticles();
- const auto& particles = pti.GetArrayOfStructs();
- auto Uarray = EulerForce[pti].array();
- auto& attri = pti.GetAttribs();
-
- auto *const fxP_ptr = attri[P_ATTR::Fx_Marker].data();
- auto *const fyP_ptr = attri[P_ATTR::Fy_Marker].data();
- auto *const fzP_ptr = attri[P_ATTR::Fz_Marker].data();
- auto *const mxP_ptr = attri[P_ATTR::Mx_Marker].data();
- auto *const myP_ptr = attri[P_ATTR::My_Marker].data();
- auto *const mzP_ptr = attri[P_ATTR::Mz_Marker].data();
- const auto *const p_ptr = particles().data();
-
- auto loc_ptr = kernel.location;
- auto dv = kernel.dv;
- auto force_index = ParticleProperties::euler_force_index;
- amrex::ParallelFor(np, [=]
- AMREX_GPU_DEVICE (int i) noexcept{
- ForceSpreading_cic(p_ptr[i], loc_ptr[0], loc_ptr[1], loc_ptr[2],
- fxP_ptr[i], fyP_ptr[i], fzP_ptr[i],
- mxP_ptr[i], myP_ptr[i], mzP_ptr[i],
- Uarray, force_index, dv, plo, dxi, type);
- });
- }
- //barrier for sync;
- amrex::ParallelDescriptor::Barrier();
-
- using pc = mParticleContainer::SuperParticleType;
- // Each Processor
- auto fx = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fx_Marker);});
- auto fy = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fy_Marker);});
- auto fz = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fz_Marker);});
- auto mx = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Mx_Marker);});
- auto my = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::My_Marker);});
- auto mz = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Mz_Marker);});
- // MPI sum reduce -> current particle all IB force and moment
- amrex::ParallelAllReduce::Sum(fx, ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(fy, ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(fz, ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(mx, ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(my, ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(mz, ParallelDescriptor::Communicator());
-
- kernel.ib_force = {fx, fy, fz};
- kernel.ib_moment = {mx, my, mz};
-
- EulerForce.SumBoundary(ParticleProperties::euler_force_index, 3, gm.periodicity());
-
- // if (false) {
- // // Check the Multifab
- // // Open a file stream for output
- // std::ofstream outFile("EulerForce.txt");
-
- // // Check the Multifab
- // // for (MFIter mfi(EulerForce, TilingIfNotGPU()); mfi.isValid(); ++mfi)
- // for (MFIter mfi(EulerForce, TilingIfNotGPU()); mfi.isValid(); ++mfi)
- // {
- // const Box& bx = mfi.validbox();
- // outFile << "Box: " << bx << "\n"
- // << "From: (" << bx.smallEnd(0) << ", " << bx.smallEnd(1) << ", " << bx.smallEnd(2) << ") "
- // << "To: (" << bx.bigEnd(0) << ", " << bx.bigEnd(1) << ", " << bx.bigEnd(2) << ")\n";
-
- // Array4 const& a = EulerForce[mfi].array();
-
- // // CPU context or illustrative purposes only
- // for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
- // for (int j = bx.smallEnd(1); j <= bx.bigEnd(1); ++j) {
- // for (int i = bx.smallEnd(0); i <= bx.bigEnd(0); ++i) {
- // // This print statement is for demonstration and should not be used in actual GPU code.
- // outFile << "Processing i: " << i << ", j: " << j << ", k: " << k << " " << a(i,j,k,0) << " " << a(i,j,k,1) << " " << a(i,j,k,2) << "\n";
- // }
- // }
- // }
- // }
-
- // // Close the file when done
- // outFile.close();
- // }
-
-}
-
-void mParticle::ResetLargrangianPoints()
-{
- if (verbose) amrex::Print() << "\tmParticle::ResetLargrangianPoints\n";
-
- for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
- const Long np = pti.numParticles();
- auto& attri = pti.GetAttribs();
-
- auto *const vUP_ptr = attri[P_ATTR::U_Marker].data();
- auto *const vVP_ptr = attri[P_ATTR::V_Marker].data();
- auto *const vWP_ptr = attri[P_ATTR::W_Marker].data();
- auto *const fxP_ptr = attri[P_ATTR::Fx_Marker].data();
- auto *const fyP_ptr = attri[P_ATTR::Fy_Marker].data();
- auto *const fzP_ptr = attri[P_ATTR::Fz_Marker].data();
- auto *const mxP_ptr = attri[P_ATTR::Mx_Marker].data();
- auto *const myP_ptr = attri[P_ATTR::My_Marker].data();
- auto *const mzP_ptr = attri[P_ATTR::Mz_Marker].data();
- amrex::ParallelFor(np, [=]
- AMREX_GPU_DEVICE (int i) noexcept{
- vUP_ptr[i] = 0.0;
- vVP_ptr[i] = 0.0;
- vWP_ptr[i] = 0.0;
- fxP_ptr[i] = 0.0;
- fyP_ptr[i] = 0.0;
- fzP_ptr[i] = 0.0;
- mxP_ptr[i] = 0.0;
- myP_ptr[i] = 0.0;
- mzP_ptr[i] = 0.0;
- });
- }
-}
-
-void mParticle::UpdateParticles(int iStep,
- Real time,
- const MultiFab& Euler_old,
- const MultiFab& Euler,
- MultiFab& phi_nodal,
- MultiFab& pvf,
- Real dt)
-{
- if (verbose) amrex::Print() << "mParticle::UpdateParticles\n";
- // start record
- auto UpdateParticlesStart = ParallelDescriptor::second();
-
- //Particle Collision calculation
- DoParticleCollision(ParticleProperties::collision_model);
-
- MultiFab AllParticlePVF(pvf.boxArray(), pvf.DistributionMap(), pvf.nComp(), pvf.nGrow());
- AllParticlePVF.setVal(0.0);
-
- //continue condition 6DOF
- for(auto& kernel : particle_kernels){
-
- calculate_phi_nodal(phi_nodal, kernel);
- nodal_phi_to_pvf(pvf, phi_nodal);
-
- // // fixed particle
- // if( ( kernel.TL.sum() == 0 ) &&
- // ( kernel.RL.sum() == 0 ) ) {
- // amrex::Print() << "Particle (" << kernel.id << ") is fixed\n";
- // MultiFab::Add(AllParticlePVF, pvf, 0, 0, 1, 0); // do not copy ghost cell values
- // continue;
- // }
-
- int ncomp = pvf.nComp();
- int ngrow = pvf.nGrow();
- MultiFab pvf_old(pvf.boxArray(), pvf.DistributionMap(), ncomp, ngrow);
- MultiFab::Copy(pvf_old, pvf, 0, 0, ncomp, ngrow);
-
- // bool at_least_one_free_trans_motion = ( kernel.TL[0] == 2 ) ||
- // ( kernel.TL[1] == 2 ) ||
- // ( kernel.TL[2] == 2 );
- // bool at_least_one_free_rot_motion = ( kernel.RL[0] == 2 ) ||
- // ( kernel.RL[1] == 2 ) ||
- // ( kernel.RL[2] == 2 );
-
- int loop = ParticleProperties::loop_solid;
-
- while (loop > 0 && iStep > ParticleProperties::start_step) {
-
- // if(at_least_one_free_trans_motion) {
- kernel.sum_u_new.scale(0.0);
- kernel.sum_u_old.scale(0.0);
- // sum U
- CalculateSumU_cir(kernel.sum_u_new, Euler, pvf, ParticleProperties::euler_velocity_index);
- CalculateSumU_cir(kernel.sum_u_old, Euler_old, pvf_old, ParticleProperties::euler_velocity_index);
- amrex::ParallelAllReduce::Sum(kernel.sum_u_new.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(kernel.sum_u_old.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
- // }
-
- // if(at_least_one_free_rot_motion) {
- kernel.sum_t_new.scale(0.0);
- kernel.sum_t_old.scale(0.0);
- // sum T
- CalculateSumT_cir(kernel.sum_t_new, Euler, pvf, kernel.location, ParticleProperties::euler_velocity_index);
- CalculateSumT_cir(kernel.sum_t_old, Euler_old, pvf_old, kernel.location, ParticleProperties::euler_velocity_index);
- amrex::ParallelAllReduce::Sum(kernel.sum_t_new.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
- amrex::ParallelAllReduce::Sum(kernel.sum_t_old.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
- // }
-
- // 6DOF
- if(ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber()){
-
- for(auto idir : {0,1,2})
- {
- //TL
- if (kernel.TL[idir] == 0) {
- kernel.velocity[idir] = 0.0;
- }
- else if (kernel.TL[idir] == 1) {
- kernel.location[idir] = kernel.location_old[idir] + (kernel.velocity[idir] + kernel.velocity_old[idir]) * dt * 0.5;
- }
- else if (kernel.TL[idir] == 2) {
- if(!ParticleProperties::Uhlmann){
- kernel.velocity[idir] = kernel.velocity_old[idir]
- + ((kernel.sum_u_new[idir] - kernel.sum_u_old[idir]) * ParticleProperties::euler_fluid_rho / dt
- - kernel.ib_force[idir] * ParticleProperties::euler_fluid_rho
- + m_gravity[idir] * (kernel.rho - ParticleProperties::euler_fluid_rho) * kernel.Vp
- + kernel.Fcp[idir]) * dt / kernel.rho / kernel.Vp ;
- }else{
- //Uhlmann
- kernel.velocity[idir] = kernel.velocity_old[idir]
- + (ParticleProperties::euler_fluid_rho / kernel.Vp /(ParticleProperties::euler_fluid_rho - kernel.rho)*kernel.ib_force[idir]
- + m_gravity[idir]) * dt;
- }
- kernel.location[idir] = kernel.location_old[idir] + (kernel.velocity[idir] + kernel.velocity_old[idir]) * dt * 0.5;
- }
- else {
- amrex::Print() << "Particle (" << kernel.id << ") has wrong TL"<< direction_str[idir] <<" value\n";
- amrex::Abort("Stop here!");
- }
- //RL
- if (kernel.RL[idir] == 0) {
- kernel.omega[idir] = 0.0;
- }
- else if (kernel.RL[idir] == 1) {
- }
- else if (kernel.RL[idir] == 2) {
- if(!ParticleProperties::Uhlmann){
- kernel.omega[idir] = kernel.omega_old[idir]
- + ((kernel.sum_t_new[idir] - kernel.sum_t_old[idir]) * ParticleProperties::euler_fluid_rho / dt
- - kernel.ib_moment[idir] * ParticleProperties::euler_fluid_rho
- + kernel.Tcp[idir]) * dt / cal_momentum(kernel.rho, kernel.radius);
- }else{
- //Uhlmann
- kernel.omega[idir] = kernel.omega_old[idir]
- + ParticleProperties::euler_fluid_rho /(ParticleProperties::euler_fluid_rho - kernel.rho) * kernel.ib_moment[idir] * kernel.dv
- / cal_momentum(kernel.rho, kernel.radius) * kernel.rho * dt;
- }
- }
- else {
- amrex::Print() << "Particle (" << kernel.id << ") has wrong RL"<< direction_str[idir] <<" value\n";
- amrex::Abort("Stop here!");
- }
-
- }
- }
- ParallelDescriptor::Bcast(&kernel.location[0],3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.location_old[0],3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.velocity[0],3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.velocity_old[0],3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.omega[0],3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.omega_old[0],3,ParallelDescriptor::IOProcessorNumber());
-
- loop--;
-
- if (loop > 0) {
- calculate_phi_nodal(phi_nodal, kernel);
- nodal_phi_to_pvf(pvf, phi_nodal);
- }
-
- }
-
- RecordOldValue(kernel);
- MultiFab::Add(AllParticlePVF, pvf, 0, 0, 1, 0); // do not copy ghost cell values
- }
- // calculate the pvf based on the information of all particles
- MultiFab::Copy(pvf, AllParticlePVF, 0, 0, 1, pvf.nGrow());
-
- spend_time += ParallelDescriptor::second() - UpdateParticlesStart;
- ParallelDescriptor::ReduceRealMax(spend_time);
- amrex::Print() << "[DIBM] IB and update particle, step : "<< iStep <<", time : " << spend_time << "\n";
-
- int particle_write_freq = ParticleProperties::write_freq;
- if (iStep % particle_write_freq == 0) {
- for(auto kernel: particle_kernels)
- WriteIBForceAndMoment(iStep, time, dt, kernel);
- }
-
- if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 4));
-}
-
-void mParticle::DoParticleCollision(int model)
-{
- if(particle_kernels.size() < 2 ) return ;
-
- if (verbose) amrex::Print() << "\tmParticle::DoParticleCollision\n";
-
- if(ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber()){
- for(const auto& kernel : particle_kernels){
- m_Collision.InsertParticle(kernel.location, kernel.velocity, kernel.radius, kernel.rho);
- }
-
- m_Collision.takeModel(model);
-
- for(auto & particle_kernel : particle_kernels){
- particle_kernel.Fcp = m_Collision.Particles.front().preForece
- * particle_kernel.Vp * particle_kernel.rho * m_gravity.vectorLength();
- m_Collision.Particles.pop_front();
- }
- }
- for(auto& kernel : particle_kernels){
- ParallelDescriptor::Bcast(kernel.Fcp.dataPtr(), 3, ParallelDescriptor::IOProcessorNumber());
- }
-}
-
-void mParticle::ComputeLagrangianForce(Real dt,
- const kernel& kernel)
-{
-
- if (verbose) amrex::Print() << "\tmParticle::ComputeLagrangianForce\n";
-
- Real Ub = kernel.velocity[0];
- Real Vb = kernel.velocity[1];
- Real Wb = kernel.velocity[2];
- Real Px = kernel.location[0];
- Real Py = kernel.location[1];
- Real Pz = kernel.location[2];
-
- for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
- const Long np = pti.numParticles();
- auto& attri = pti.GetAttribs();
- auto const* p_ptr = pti.GetArrayOfStructs().data();
-
- auto* Up = attri[P_ATTR::U_Marker].data();
- auto* Vp = attri[P_ATTR::V_Marker].data();
- auto* Wp = attri[P_ATTR::W_Marker].data();
- auto *FxP = attri[P_ATTR::Fx_Marker].data();
- auto *FyP = attri[P_ATTR::Fy_Marker].data();
- auto *FzP = attri[P_ATTR::Fz_Marker].data();
-
- amrex::ParallelFor(np,
- [=] AMREX_GPU_DEVICE (int i) noexcept{
- auto Ur = (kernel.omega).crossProduct(RealVect(p_ptr[i].pos(0) - Px, p_ptr[i].pos(1) - Py, p_ptr[i].pos(2) - Pz));
- FxP[i] = (Ub + Ur[0] - Up[i])/dt; //
- FyP[i] = (Vb + Ur[1] - Vp[i])/dt; //
- FzP[i] = (Wb + Ur[2] - Wp[i])/dt; //
- });
- }
- if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 3));
-}
-
-void mParticle::VelocityCorrection(amrex::MultiFab &Euler, amrex::MultiFab &EulerForce, Real dt) const
-{
- if(verbose) amrex::Print() << "\tmParticle::VelocityCorrection\n";
- MultiFab::Saxpy(Euler, dt, EulerForce, ParticleProperties::euler_force_index, ParticleProperties::euler_velocity_index, 3, 0); //VelocityCorrection
-}
-
-void mParticle::RecordOldValue(kernel& kernel)
-{
- kernel.location_old = kernel.location;
- kernel.velocity_old = kernel.velocity;
- kernel.omega_old = kernel.omega;
-}
-
-void mParticle::WriteParticleFile(int index)
-{
- mContainer->WriteAsciiFile(amrex::Concatenate("particle", index));
-}
-
-void mParticle::WriteIBForceAndMoment(int step, amrex::Real time, amrex::Real dt, kernel& current_kernel)
-{
-
- if(amrex::ParallelDescriptor::MyProc() != ParallelDescriptor::IOProcessorNumber()) return;
-
- std::string file("IB_Particle_" + std::to_string(current_kernel.id) + ".csv");
- std::ofstream out_ib_force;
-
- std::string head;
- if(!fs::exists(file)){
- head = "iStep,time,X,Y,Z,Vx,Vy,Vz,Rx,Ry,Rz,Fx,Fy,Fz,Mx,My,Mz,Fcpx,Fcpy,Fcpz,Tcpx,Tcpy,Tcpz,SumUx,SumUy,SumUz,SumTx,SumTy,SumTz\n";
- }else{
- head = "";
- }
-
- out_ib_force.open(file, std::ios::app);
- if(!out_ib_force.is_open()){
- amrex::Print() << "[Particle] write particle file error , step: " << step;
- }else{
- out_ib_force << head << step << "," << time << ","
- << current_kernel.location[0] << "," << current_kernel.location[1] << "," << current_kernel.location[2] << ","
- << current_kernel.velocity[0] << "," << current_kernel.velocity[1] << "," << current_kernel.velocity[2] << ","
- << current_kernel.omega[0] << "," << current_kernel.omega[1] << "," << current_kernel.omega[2] << ","
- << current_kernel.ib_force[0] << "," << current_kernel.ib_force[1] << "," << current_kernel.ib_force[2] << ","
- << current_kernel.ib_moment[0] << "," << current_kernel.ib_moment[1] << "," << current_kernel.ib_moment[2] << ","
- << current_kernel.Fcp[0] << "," << current_kernel.Fcp[1] << "," << current_kernel.Fcp[2] << ","
- << current_kernel.Tcp[0] << "," << current_kernel.Tcp[1] << "," << current_kernel.Tcp[2] << ","
- << (current_kernel.sum_u_new[0] - current_kernel.sum_u_old[0])/dt << ","
- << (current_kernel.sum_u_new[1] - current_kernel.sum_u_old[1])/dt << ","
- << (current_kernel.sum_u_new[2] - current_kernel.sum_u_old[2])/dt << ","
- << (current_kernel.sum_t_new[0] - current_kernel.sum_t_old[0])/dt << ","
- << (current_kernel.sum_t_new[1] - current_kernel.sum_t_old[1])/dt << ","
- << (current_kernel.sum_t_new[2] - current_kernel.sum_t_old[2])/dt << "\n";
- }
- out_ib_force.close();
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-/* Particles member function */
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-void Particles::create_particles(const Geometry &gm,
- const DistributionMapping & dm,
- const BoxArray & ba)
-{
- amrex::Print() << "[Particle] : create Particle Container\n";
- if(particle->mContainer != nullptr){
- delete particle->mContainer;
- particle->mContainer = nullptr;
- }
- particle->mContainer = new mParticleContainer(gm, dm, ba);
-
- //get particle tile
- std::pair key{0,0};
- auto& particleTileTmp = particle->mContainer->GetParticles(0)[key];
- //insert markers
- if ( ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber() ) {
- //insert particle's markers
- //Real phiK = 0;
- for(int marker_index = 0; marker_index < particle->particle_kernels[0].ml; marker_index++){
- //insert code
- mParticleContainer::ParticleType markerP;
- markerP.id() = marker_index + 1;
- markerP.cpu() = ParallelDescriptor::MyProc();
- markerP.pos(0) = particle->particle_kernels[0].location[0];
- markerP.pos(1) = particle->particle_kernels[0].location[1];
- markerP.pos(2) = particle->particle_kernels[0].location[2];
-
- std::array Marker_attr;
- Marker_attr[U_Marker] = 0.0;
- Marker_attr[V_Marker] = 0.0;
- Marker_attr[W_Marker] = 0.0;
- Marker_attr[Fx_Marker] = 0.0;
- Marker_attr[Fy_Marker] = 0.0;
- Marker_attr[Fz_Marker] = 0.0;
-
- particleTileTmp.push_back(markerP);
- particleTileTmp.push_back_real(Marker_attr);
- }
- }
- particle->mContainer->Redistribute(); // Still needs to redistribute here!
-
- ParticleProperties::plo = gm.ProbLoArray();
- ParticleProperties::phi = gm.ProbHiArray();
- ParticleProperties::dx = gm.CellSizeArray();
-}
-
-mParticle* Particles::get_particles()
-{
- return particle;
-}
-
-
-void Particles::init_particle(Real gravity, Real h)
-{
- amrex::Print() << "[Particle] : create Particle's kernel\n";
- particle = new mParticle;
- if(particle != nullptr){
- isInitial = true;
- particle->InitParticles(
- ParticleProperties::_x,
- ParticleProperties::_y,
- ParticleProperties::_z,
- ParticleProperties::_rho,
- ParticleProperties::Vx,
- ParticleProperties::Vy,
- ParticleProperties::Vz,
- ParticleProperties::Ox,
- ParticleProperties::Oy,
- ParticleProperties::Oz,
- ParticleProperties::TLX,
- ParticleProperties::TLY,
- ParticleProperties::TLZ,
- ParticleProperties::RLX,
- ParticleProperties::RLY,
- ParticleProperties::RLZ,
- ParticleProperties::_radius,
- h,
- gravity,
- ParticleProperties::verbose);
- }
-
-}
-
-void Particles::Restart(Real gravity, Real h, int iStep)
-{
- amrex::Print() << "[Particle] : restart Particle's kernel, step :" << iStep << "\n"
- << "\tstart read particle csv file , default name is IB_Particle_x.csv\n"
- << "\tdo not delete those file before \"restart\"\n\n";
- delete particle;
- particle = new mParticle;
- particle->InitParticles(
- ParticleProperties::_x,
- ParticleProperties::_y,
- ParticleProperties::_z,
- ParticleProperties::_rho,
- ParticleProperties::Vx,
- ParticleProperties::Vy,
- ParticleProperties::Vz,
- ParticleProperties::Ox,
- ParticleProperties::Oy,
- ParticleProperties::Oz,
- ParticleProperties::TLX,
- ParticleProperties::TLY,
- ParticleProperties::TLZ,
- ParticleProperties::RLX,
- ParticleProperties::RLY,
- ParticleProperties::RLZ,
- ParticleProperties::_radius,
- h,
- gravity,
- ParticleProperties::verbose);
- //deal in IO processor
- //start read csv file
- for(auto& kernel : particle->particle_kernels){
- //filename
- if(amrex::ParallelDescriptor::MyProc() == amrex::ParallelDescriptor::IOProcessorNumber()){
- std::string fileName = "IB_Particle_" + std::to_string(kernel.id) + ".csv";
- std::string tmpfile = "tmp" + fileName;
- //file stream
- std::ifstream particle_data(fileName);
- std::ofstream particle_file(tmpfile);
- // open state
- if(!particle_data.is_open() || !particle_file.is_open()){
- amrex::Abort("\tCan not open particle file : " + fileName);
- }
- std::string lineData;
- int line{0};
- while(std::getline(particle_data, lineData)){
- line++;
- particle_file << lineData << "\n";
- if(line <= iStep) {
- continue;
- }
- //old location
- //iStep,time,X,Y,Z,Vx,Vy,Vz,Rx,Ry,Rz,Fx,Fy,Fz,Mx,My,Mz,Fcpx,Fcpy,Fcpz,Tcpx,Tcpy,Tcpz
- if(line == iStep + 1){
- std::stringstream ss(lineData);
- std::string data;
- std::vector dataStruct;
- while(std::getline(ss, data, ',')){
- dataStruct.emplace_back(std::stod(data));
- }
- kernel.location[0] = dataStruct[2];
- kernel.location[1] = dataStruct[3];
- kernel.location[2] = dataStruct[4];
- kernel.velocity[0] = dataStruct[5];
- kernel.velocity[1] = dataStruct[6];
- kernel.velocity[2] = dataStruct[7];
- kernel.omega[0] = dataStruct[8];
- kernel.omega[1] = dataStruct[9];
- kernel.omega[2] = dataStruct[10];
-
- kernel.location_old = kernel.location;
- kernel.velocity_old = kernel.velocity;
- kernel.omega_old = kernel.omega;
- break;
- }
- else
- break;
- }
- particle_data.close();
- particle_file.close();
- std::remove(fileName.c_str());
- std::rename(tmpfile.c_str(), fileName.c_str());
- }
- ParallelDescriptor::Bcast(&kernel.location[0], 3, ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.velocity[0], 3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.omega[0], 3,ParallelDescriptor::IOProcessorNumber());
-
- ParallelDescriptor::Bcast(&kernel.location_old[0], 3, ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.velocity_old[0], 3,ParallelDescriptor::IOProcessorNumber());
- ParallelDescriptor::Bcast(&kernel.omega_old[0], 3,ParallelDescriptor::IOProcessorNumber());
- }
-
- isInitial = true;
-}
-
-void Particles::Initialize()
-{
- ParmParse pp("particle");
-
- std::string particle_inputfile;
- std::string particle_init_file;
- pp.get("input",particle_inputfile);
-
- if(!particle_inputfile.empty()){
- ParmParse p_file(particle_inputfile);
- p_file.query("init", particle_init_file);
- p_file.getarr("x", ParticleProperties::_x);
- p_file.getarr("y", ParticleProperties::_y);
- p_file.getarr("z", ParticleProperties::_z);
- p_file.getarr("rho", ParticleProperties::_rho);
- p_file.getarr("velocity_x", ParticleProperties::Vx);
- p_file.getarr("velocity_y", ParticleProperties::Vy);
- p_file.getarr("velocity_z", ParticleProperties::Vz);
- p_file.getarr("omega_x", ParticleProperties::Ox);
- p_file.getarr("omega_y", ParticleProperties::Oy);
- p_file.getarr("omega_z", ParticleProperties::Oz);
- p_file.getarr("TLX", ParticleProperties::TLX);
- p_file.getarr("TLY", ParticleProperties::TLY);
- p_file.getarr("TLZ", ParticleProperties::TLZ);
- p_file.getarr("RLX", ParticleProperties::RLX);
- p_file.getarr("RLY", ParticleProperties::RLY);
- p_file.getarr("RLZ", ParticleProperties::RLZ);
- p_file.getarr("radius", ParticleProperties::_radius);
- p_file.query("RD", ParticleProperties::rd);
- p_file.query("LOOP_NS", ParticleProperties::loop_ns);
- p_file.query("LOOP_SOLID", ParticleProperties::loop_solid);
- p_file.query("verbose", ParticleProperties::verbose);
- p_file.query("start_step", ParticleProperties::start_step);
- p_file.query("Uhlmann", ParticleProperties::Uhlmann);
- p_file.query("collision_model", ParticleProperties::collision_model);
- p_file.query("write_freq", ParticleProperties::write_freq);
-
- ParmParse ns("ns");
- ns.get("fluid_rho", ParticleProperties::euler_fluid_rho);
-
- ParmParse level_parse("amr");
- level_parse.get("max_level", ParticleProperties::euler_finest_level);
-
- ParmParse geometry_parse("geometry");
- geometry_parse.getarr("prob_lo", ParticleProperties::GLO);
- geometry_parse.getarr("prob_hi", ParticleProperties::GHI);
- amrex::Print() << "[Particle] : Reading partilces cfg file : " << particle_inputfile << "\n"
- << " Particle's level : " << ParticleProperties::euler_finest_level << "\n";
-
- if(!particle_init_file.empty()){
- ParticleProperties::init_particle_from_file = true;
- //clear particle position container
- ParticleProperties::_x.clear();
- ParticleProperties::_y.clear();
- ParticleProperties::_z.clear();
- // parse particle's location data
- std::ifstream init_particle(particle_init_file);
- std::string line_data;
- while(std::getline(init_particle, line_data)){
- // id x_location y_location z_location
- std::istringstream line(line_data);
- std::vector str_tokne;
- std::string token;
- while(line >> token){
- str_tokne.push_back(token);
- }
-
- ParticleProperties::_x.push_back(std::stod(str_tokne[0]));
- ParticleProperties::_y.push_back(std::stod(str_tokne[1]));
- ParticleProperties::_z.push_back(std::stod(str_tokne[2]));
- }
- ParticleProperties::_x.shrink_to_fit();
- ParticleProperties::_y.shrink_to_fit();
- ParticleProperties::_z.shrink_to_fit();
- amrex::Print() << " initial Particle by file : " << particle_init_file
- << " particle's size : " << ParticleProperties::_x.size() << "\n";
- }
-
- }else {
- amrex::Abort("[Particle] : can't read particles settings, pls check your config file \"particle.input\"");
- }
-}
-
-int Particles::ParticleFinestLevel()
-{
- return ParticleProperties::euler_finest_level;
-}
+// SPDX-FileCopyrightText: 2023 - 2025 Yadong Zeng & ZhuXu Li<1246206018@qq.com>
+//
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+namespace fs = std::filesystem;
+
+#define GHOST_CELLS 2
+
+using namespace amrex;
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* global variable */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+#define LOCAL_LEVEL 0
+
+const Vector direction_str{"X","Y","Z"};
+
+namespace ParticleProperties{
+ Vector _x{}, _y{}, _z{}, _rho{};
+ Vector Vx{}, Vy{}, Vz{};
+ Vector Ox{}, Oy{}, Oz{};
+ Vector _radius;
+ Real rd{0.0};
+ Vector TLX{}, TLY{},TLZ{},RLX{},RLY{},RLZ{};
+ int euler_finest_level{0};
+ int euler_velocity_index{0};
+ int euler_force_index{0};
+ Real euler_fluid_rho{0.0};
+ int verbose{0};
+ int loop_ns{2};
+ int loop_solid{1};
+ int Uhlmann{0};
+
+ Vector GLO, GHI;
+ int start_step{-1};
+ int collision_model{0};
+
+ int write_freq{1};
+ bool init_particle_from_file{false};
+
+ GpuArray plo{0.0,0.0,0.0}, phi{0.0,0.0,0.0}, dx{0.0, 0.0, 0.0};
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* other function */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void nodal_phi_to_pvf(MultiFab& pvf, const MultiFab& phi_nodal)
+{
+
+ // amrex::Print() << "In the nodal_phi_to_pvf\n";
+
+ pvf.setVal(0.0);
+
+ // Only set the valid cells of pvf
+#ifdef AMREX_USE_OMP
+#pragma omp parallel if (Gpu::notInLaunchRegion())
+#endif
+ for (MFIter mfi(pvf,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+ {
+ const Box& bx = mfi.tilebox();
+ auto const& pvffab = pvf.array(mfi);
+ auto const& pnfab = phi_nodal.array(mfi);
+ amrex::ParallelFor(bx, [pvffab, pnfab]
+ AMREX_GPU_DEVICE(int i, int j, int k) noexcept
+ {
+ Real num = 0.0;
+ for(int kk=k; kk<=k+1; kk++) {
+ for(int jj=j; jj<=j+1; jj++) {
+ for(int ii=i; ii<=i+1; ii++) {
+ num += (-pnfab(ii,jj,kk)) * nodal_phi_to_heavi(-pnfab(ii,jj,kk));
+ }
+ }
+ }
+ Real deo = 0.0;
+ for(int kk=k; kk<=k+1; kk++) {
+ for(int jj=j; jj<=j+1; jj++) {
+ for(int ii=i; ii<=i+1; ii++) {
+ deo += std::abs(pnfab(ii,jj,kk));
+ }
+ }
+ }
+ pvffab(i,j,k) = num / (deo + 1.e-12);
+ });
+ }
+
+}
+
+void calculate_phi_nodal(MultiFab& phi_nodal, kernel& current_kernel)
+{
+ phi_nodal.setVal(0.0);
+
+ amrex::Real Xp = current_kernel.location[0];
+ amrex::Real Yp = current_kernel.location[1];
+ amrex::Real Zp = current_kernel.location[2];
+ amrex::Real Rp = current_kernel.radius;
+
+ // Only set the valid cells of phi_nodal
+ for (MFIter mfi(phi_nodal,TilingIfNotGPU()); mfi.isValid(); ++mfi)
+ {
+ const Box& bx = mfi.tilebox();
+ auto const& pnfab = phi_nodal.array(mfi);
+ auto dx = ParticleProperties::dx;
+ auto plo = ParticleProperties::plo;
+ amrex::ParallelFor(bx, [=]
+ AMREX_GPU_DEVICE(int i, int j, int k) noexcept
+ {
+ Real Xn = i * dx[0] + plo[0];
+ Real Yn = j * dx[1] + plo[1];
+ Real Zn = k * dx[2] + plo[2];
+
+ pnfab(i,j,k) = std::sqrt( (Xn - Xp)*(Xn - Xp)
+ + (Yn - Yp)*(Yn - Yp) + (Zn - Zp)*(Zn - Zp)) - Rp;
+ pnfab(i,j,k) = pnfab(i,j,k) / Rp;
+
+ }
+ );
+ }
+}
+
+// May use ParReduce later, https://amrex-codes.github.io/amrex/docs_html/GPU.html#multifab-reductions
+void CalculateSumU_cir (RealVect& sum,
+ const MultiFab& E,
+ const MultiFab& pvf,
+ int EulerVelIndex)
+{
+ auto const& E_data = E.const_arrays();
+ auto const& pvf_data = pvf.const_arrays();
+ const Real d = Math::powi<3>(ParticleProperties::dx[0]);
+ amrex::GpuTuple tmpSum = ParReduce(TypeList{}, TypeList{},E, IntVect{0},
+ [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept -> amrex::GpuTuple{
+ auto E_ = E_data[box_no];
+ auto pvf_ = pvf_data[box_no];
+ return {
+ E_(i, j, k, EulerVelIndex ) * d * pvf_(i,j,k),
+ E_(i, j, k, EulerVelIndex + 1) * d * pvf_(i,j,k),
+ E_(i, j, k, EulerVelIndex + 2) * d * pvf_(i,j,k)
+ };
+ });
+ sum[0] = amrex::get<0>(tmpSum);
+ sum[1] = amrex::get<1>(tmpSum);
+ sum[2] = amrex::get<2>(tmpSum);
+}
+
+void CalculateSumT_cir (RealVect& sum,
+ const MultiFab& E,
+ const MultiFab& pvf,
+ const RealVect pLoc,
+ int EulerVelIndex)
+{
+ auto plo = ParticleProperties::plo;
+ auto dx = ParticleProperties::dx;
+
+ auto const& E_data = E.const_arrays();
+ auto const& pvf_data = pvf.const_arrays();
+ const Real d = Math::powi<3>(ParticleProperties::dx[0]);
+ amrex::GpuTuple tmpSum = ParReduce(TypeList{}, TypeList{},E, IntVect{0},
+ [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept -> amrex::GpuTuple{
+ auto E_ = E_data[box_no];
+ auto pvf_ = pvf_data[box_no];
+
+ Real x = plo[0] + i*dx[0] + 0.5*dx[0];
+ Real y = plo[1] + j*dx[1] + 0.5*dx[1];
+ Real z = plo[2] + k*dx[2] + 0.5*dx[2];
+
+ Real vx = E_(i, j, k, EulerVelIndex );
+ Real vy = E_(i, j, k, EulerVelIndex + 1);
+ Real vz = E_(i, j, k, EulerVelIndex + 2);
+
+ RealVect tmp = RealVect(x - pLoc[0], y - pLoc[1], z - pLoc[2]).crossProduct(RealVect(vx, vy, vz));
+
+ return {
+ tmp[0] * d * pvf_(i, j, k),
+ tmp[1] * d * pvf_(i, j, k),
+ tmp[2] * d * pvf_(i, j, k)
+ };
+ });
+ sum[0] = amrex::get<0>(tmpSum);
+ sum[1] = amrex::get<1>(tmpSum);
+ sum[2] = amrex::get<2>(tmpSum);
+}
+
+[[nodiscard]] AMREX_FORCE_INLINE
+Real cal_momentum(Real rho, Real radius)
+{
+ return 8.0 * Math::pi() * rho * Math::powi<5>(radius) / 15.0;
+}
+
+AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+void deltaFunction(Real xf, Real xp, Real h, Real& value, DELTA_FUNCTION_TYPE type)
+{
+ Real rr = amrex::Math::abs(( xf - xp ) / h);
+
+ switch (type) {
+ case DELTA_FUNCTION_TYPE::FOUR_POINT_IB:
+ if(rr >= 0 && rr < 1 ){
+ value = 1.0 / 8.0 * ( 3.0 - 2.0 * rr + std::sqrt( 1.0 + 4.0 * rr - 4 * Math::powi<2>(rr))) / h;
+ }else if (rr >= 1 && rr < 2) {
+ value = 1.0 / 8.0 * ( 5.0 - 2.0 * rr - std::sqrt( -7.0 + 12.0 * rr - 4 * Math::powi<2>(rr))) / h;
+ }else {
+ value = 0;
+ }
+ break;
+ case DELTA_FUNCTION_TYPE::THREE_POINT_IB:
+ if(rr >= 0.5 && rr < 1.5){
+ value = 1.0 / 6.0 * ( 5.0 - 3.0 * rr - std::sqrt( - 3.0 * Math::powi<2>( 1 - rr) + 1.0 )) / h;
+ }else if (rr >= 0 && rr < 0.5) {
+ value = 1.0 / 3.0 * ( 1.0 + std::sqrt( 1.0 - 3 * Math::powi<2>(rr))) / h;
+ }else {
+ value = 0;
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* mParticle member function */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+//loop all particels
+void mParticle::InteractWithEuler(MultiFab &EulerVel,
+ MultiFab &EulerForce,
+ Real dt,
+ DELTA_FUNCTION_TYPE type)
+{
+ if (verbose) amrex::Print() << "[Particle] mParticle::InteractWithEuler\n";
+ // clear time , start record
+ spend_time = 0;
+ auto InteractWithEulerStart = ParallelDescriptor::second();
+
+ MultiFab EulerForceTmp(EulerForce.boxArray(), EulerForce.DistributionMap(), 3, EulerForce.nGrow());
+ //clean preStep's IB_porperties
+ for(auto& kernel : particle_kernels) {
+ kernel.ib_force.scale(0.0);
+ kernel.ib_moment.scale(0.0);
+ }
+
+ //for 1 -> Ns
+ int loop = ParticleProperties::loop_ns;
+ BL_ASSERT(loop > 0);
+ while(loop > 0){
+ if(verbose) amrex::Print() << "[Particle] Ns loop index : " << loop << "\n";
+
+ EulerForce.setVal(0.0);
+
+ for(kernel& kernel : particle_kernels){
+ InitialWithLargrangianPoints(kernel); // Initialize markers for a specific particle
+ ResetLargrangianPoints();
+ EulerForceTmp.setVal(0.0);
+ auto ib_force = kernel.ib_force;
+ auto ib_moment = kernel.ib_moment;
+ kernel.ib_force.scale(0.0); // clear kernel ib_force
+ kernel.ib_moment.scale(0.0); // clear kernel ib_moment
+
+ VelocityInterpolation(EulerVel, type);
+ ComputeLagrangianForce(dt, kernel);
+ ForceSpreading(EulerForceTmp, kernel, type);
+ MultiFab::Add(EulerForce, EulerForceTmp, 0, 0, 3, EulerForce.nGrow());
+
+ kernel.ib_force += ib_force;
+ kernel.ib_moment += ib_moment;
+ }
+ VelocityCorrection(EulerVel, EulerForce, dt);
+ loop--;
+ }
+ spend_time += ParallelDescriptor::second() - InteractWithEulerStart;
+}
+
+void mParticle::InitParticles(const Vector& x,
+ const Vector& y,
+ const Vector& z,
+ const Vector& rho_s,
+ const Vector& Vx,
+ const Vector& Vy,
+ const Vector& Vz,
+ const Vector& Ox,
+ const Vector& Oy,
+ const Vector& Oz,
+ const Vector& TLXt,
+ const Vector& TLYt,
+ const Vector& TLZt,
+ const Vector& RLXt,
+ const Vector& RLYt,
+ const Vector& RLZt,
+ const Vector& radius,
+ Real h,
+ Real gravity,
+ int _verbose)
+{
+ verbose = _verbose;
+ if (verbose) amrex::Print() << "[Particle] mParticle::InitParticles\n";
+
+ m_gravity[2] = gravity;
+
+ //pre judge
+ if(!((x.size() == y.size()) && (x.size() == z.size()))){
+ Print() << "particle's position container are all different size";
+ return;
+ }
+
+ //all the particles have different radius
+ for(int index = 0; index < x.size(); index++){
+ int real_index;
+ // if initial with input file, initialized by [0] data
+ if(ParticleProperties::init_particle_from_file){
+ real_index = 0;
+ }else{
+ real_index = index;
+ }
+
+ kernel mKernel;
+ mKernel.id = index + 1;
+ mKernel.location[0] = x[index];
+ mKernel.location[1] = y[index];
+ mKernel.location[2] = z[index];
+ mKernel.velocity[0] = Vx[real_index];
+ mKernel.velocity[1] = Vy[real_index];
+ mKernel.velocity[2] = Vz[real_index];
+ mKernel.omega[0] = Ox[real_index];
+ mKernel.omega[1] = Oy[real_index];
+ mKernel.omega[2] = Oz[real_index];
+
+ // use current state to initialize old state
+ mKernel.location_old = mKernel.location;
+ mKernel.velocity_old = mKernel.velocity;
+ mKernel.omega_old = mKernel.omega;
+
+ mKernel.TL[0] = TLXt[real_index];
+ mKernel.TL[1] = TLYt[real_index];
+ mKernel.TL[2] = TLZt[real_index];
+ mKernel.RL[0] = RLXt[real_index];
+ mKernel.RL[1] = RLYt[real_index];
+ mKernel.RL[2] = RLZt[real_index];
+ mKernel.rho = rho_s[real_index];
+ mKernel.radius = radius[real_index];
+ mKernel.Vp = Math::pi() * 4 / 3 * Math::powi<3>(radius[real_index]);
+
+ //int Ml = static_cast( Math::pi() / 3 * (12 * Math::powi<2>(mKernel.radius / h)));
+ //Real dv = Math::pi() * h / 3 / Ml * (12 * mKernel.radius * mKernel.radius + h * h);
+ int Ml = static_cast((amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd - 0.5) * h)
+ - amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd + 0.5) * h))/(3.*h*h*h/4./Math::pi()));
+ Real dv = (amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd - 0.5) * h)
+ - amrex::Math::powi<3>(mKernel.radius - (ParticleProperties::rd + 0.5) * h))/(3.*Ml/4./Math::pi());
+ mKernel.ml = Ml;
+ mKernel.dv = dv;
+ if( Ml > max_largrangian_num ) max_largrangian_num = Ml;
+
+ Real phiK = 0;
+ for(int marker_index = 0; marker_index < Ml; marker_index++){
+ Real Hk = -1.0 + 2.0 * (marker_index) / ( Ml - 1.0);
+ Real thetaK = std::acos(Hk);
+ if(marker_index == 0 || marker_index == (Ml - 1)){
+ phiK = 0;
+ }else {
+ phiK = std::fmod( phiK + 3.809 / std::sqrt(Ml) / std::sqrt( 1 - Math::powi<2>(Hk)) , 2 * Math::pi());
+ }
+ mKernel.phiK.push_back(phiK);
+ mKernel.thetaK.push_back(thetaK);
+ }
+
+ particle_kernels.emplace_back(mKernel);
+
+ if (verbose) amrex::Print() << "h: " << h << ", Ml: " << Ml << ", D: " << Math::powi<3>(h) << " gravity : " << gravity << "\n"
+ << "Kernel : " << index << ": Location (" << x[index] << ", " << y[index] << ", " << z[index]
+ << "), Velocity : (" << mKernel.velocity[0] << ", " << mKernel.velocity[1] << ", "<< mKernel.velocity[2]
+ << "), Radius: " << mKernel.radius << ", Ml: " << Ml << ", dv: " << dv << ", Rho: " << mKernel.rho << "\n";
+ }
+ //collision box generate
+ m_Collision.SetGeometry(RealVect(ParticleProperties::GLO), RealVect(ParticleProperties::GHI),particle_kernels[0].radius, h);
+}
+
+void mParticle::InitialWithLargrangianPoints(const kernel& current_kernel){
+
+ if (verbose) amrex::Print() << "mParticle::InitialWithLargrangianPoints\n";
+ for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
+ const Long np = pti.numParticles();
+ if(np == 0) continue;
+ auto *particles = pti.GetArrayOfStructs().data();
+
+ const auto location = current_kernel.location;
+ const auto radius = current_kernel.radius;
+ const auto* phiK = current_kernel.phiK.dataPtr();
+ const auto* thetaK = current_kernel.thetaK.dataPtr();
+
+ amrex::ParallelFor( np, [=]
+ AMREX_GPU_DEVICE (int i) noexcept {
+ auto id = particles[i].id();
+ particles[i].pos(0) = location[0] + radius * std::sin(thetaK[id - 1]) * std::cos(phiK[id - 1]);
+ particles[i].pos(1) = location[1] + radius * std::sin(thetaK[id - 1]) * std::sin(phiK[id - 1]);
+ particles[i].pos(2) = location[2] + radius * std::cos(thetaK[id - 1]);
+ }
+ );
+ }
+ // Redistribute the markers after updating their locations
+ mContainer->Redistribute();
+ if (verbose) {
+ amrex::Print() << "[particle] : particle num :" << mContainer->TotalNumberOfParticles() << "\n";
+ mContainer->WriteAsciiFile(amrex::Concatenate("particle", 1));
+ }
+}
+
+template >
+AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+void VelocityInterpolation_cir(int p_iter, P const& p, Real& Up, Real& Vp, Real& Wp,
+ Array4 const& E, int EulerVIndex,
+ const int *lo, const int *hi,
+ GpuArray const& plo,
+ GpuArray const& dx,
+ DELTA_FUNCTION_TYPE type)
+{
+
+ //std::cout << "lo " << lo[0] << " " << lo[1] << " "<< lo[2] << "\n";
+ //std::cout << "hi " << hi[0] << " " << hi[1] << " "<< hi[2] << "\n";
+
+ const Real d = AMREX_D_TERM(dx[0], *dx[1], *dx[2]);
+
+ const Real lx = (p.pos(0) - plo[0]) / dx[0]; // x
+ const Real ly = (p.pos(1) - plo[1]) / dx[1]; // y
+ const Real lz = (p.pos(2) - plo[2]) / dx[2]; // z
+
+ int i = static_cast(Math::floor(lx)); // i
+ int j = static_cast(Math::floor(ly)); // j
+ int k = static_cast(Math::floor(lz)); // k
+
+ //std::cout << "p_iter " << p_iter << " p.pos(0): " << p.pos(0) << " p.pos(1): " << p.pos(1) << " p.pos(2): " << p.pos(2) << "\n";
+
+ // std::cout << "d: " << d << "\n"
+ // << "lx: " << lx << ", ly: " << ly << ", lz: " << lz << "\n"
+ // << "i: " << i << ", j: " << j << ", k: " << k << std::endl;
+
+ Up = 0;
+ Vp = 0;
+ Wp = 0;
+ //Euler to largrangian
+ for(int ii = -2; ii < 3; ii++){
+ for(int jj = -2; jj < 3; jj++){
+ for(int kk = -2; kk < 3; kk ++){
+ Real tU, tV, tW;
+ const Real xi = plo[0] + (i + ii) * dx[0] + dx[0]/2;
+ const Real yj = plo[1] + (j + jj) * dx[1] + dx[1]/2;
+ const Real kz = plo[2] + (k + kk) * dx[2] + dx[2]/2;
+ deltaFunction( p.pos(0), xi, dx[0], tU, type);
+ deltaFunction( p.pos(1), yj, dx[1], tV, type);
+ deltaFunction( p.pos(2), kz, dx[2], tW, type);
+ const Real delta_value = tU * tV * tW;
+ Up += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex ) * d;
+ Vp += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex + 1) * d;
+ Wp += delta_value * E(i + ii, j + jj, k + kk, EulerVIndex + 2) * d;
+ }
+ }
+ }
+}
+
+void mParticle::VelocityInterpolation(MultiFab &EulerVel,
+ DELTA_FUNCTION_TYPE type)//
+{
+ if (verbose) amrex::Print() << "\tmParticle::VelocityInterpolation\n";
+
+ //amrex::Print() << "euler_finest_level " << euler_finest_level << std::endl;
+ const auto& gm = mContainer->GetParGDB()->Geom(LOCAL_LEVEL);
+ auto plo = gm.ProbLoArray();
+ auto dx = gm.CellSizeArray();
+ // attention
+ // velocity ghost cells will be up-to-date
+ EulerVel.FillBoundary(ParticleProperties::euler_velocity_index, 3, gm.periodicity());
+
+ const int EulerVelocityIndex = ParticleProperties::euler_velocity_index;
+
+ for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
+
+ const Box& box = pti.validbox();
+
+ auto& particles = pti.GetArrayOfStructs();
+ auto *p_ptr = particles.data();
+ const Long np = pti.numParticles();
+
+ auto& attri = pti.GetAttribs();
+ auto* Up = attri[P_ATTR::U_Marker].data();
+ auto* Vp = attri[P_ATTR::V_Marker].data();
+ auto* Wp = attri[P_ATTR::W_Marker].data();
+ const auto& E = EulerVel.array(pti);
+
+ amrex::ParallelFor(np, [=]
+ AMREX_GPU_DEVICE (int i) noexcept{
+ VelocityInterpolation_cir(i, p_ptr[i], Up[i], Vp[i], Wp[i], E, EulerVelocityIndex, box.loVect(), box.hiVect(), plo, dx, type);
+ });
+ }
+ if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 2));
+ //amrex::Abort("stop here!");
+}
+
+template
+AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+void ForceSpreading_cic (P const& p,
+ Real Px,
+ Real Py,
+ Real Pz,
+ ParticleReal& fxP,
+ ParticleReal& fyP,
+ ParticleReal& fzP,
+ ParticleReal& mxP,
+ ParticleReal& myP,
+ ParticleReal& mzP,
+ Array4 const& E,
+ int EulerForceIndex,
+ Real dv,
+ GpuArray const& plo,
+ GpuArray const& dx,
+ DELTA_FUNCTION_TYPE type)
+{
+ //const Real d = AMREX_D_TERM(dx[0], *dx[1], *dx[2]);
+ //plo to ii jj kk
+ Real lx = (p.pos(0) - plo[0]) / dx[0];
+ Real ly = (p.pos(1) - plo[1]) / dx[1];
+ Real lz = (p.pos(2) - plo[2]) / dx[2];
+
+ int i = static_cast(Math::floor(lx));
+ int j = static_cast(Math::floor(ly));
+ int k = static_cast(Math::floor(lz));
+ fxP *= dv;
+ fyP *= dv;
+ fzP *= dv;
+ RealVect moment = RealVect((p.pos(0) - Px), (p.pos(1) - Py), (p.pos(2) - Pz)).crossProduct(RealVect(fxP, fyP, fzP));
+ mxP = moment[0];
+ myP = moment[1];
+ mzP = moment[2];
+ //lagrangian to Euler
+ for(int ii = -2; ii < +3; ii++){
+ for(int jj = -2; jj < +3; jj++){
+ for(int kk = -2; kk < +3; kk ++){
+ Real tU, tV, tW;
+ const Real xi =plo[0] + (i + ii) * dx[0] + dx[0]/2;
+ const Real yj =plo[1] + (j + jj) * dx[1] + dx[1]/2;
+ const Real kz =plo[2] + (k + kk) * dx[2] + dx[2]/2;
+ deltaFunction( p.pos(0), xi, dx[0], tU, type);
+ deltaFunction( p.pos(1), yj, dx[1], tV, type);
+ deltaFunction( p.pos(2), kz, dx[2], tW, type);
+ Real delta_value = tU * tV * tW;
+ Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex ), delta_value * fxP);
+ Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex+1), delta_value * fyP);
+ Gpu::Atomic::AddNoRet(&E(i + ii, j + jj, k + kk, EulerForceIndex+2), delta_value * fzP);
+ }
+ }
+ }
+}
+
+void mParticle::ForceSpreading(MultiFab & EulerForce,
+ kernel& kernel,
+ DELTA_FUNCTION_TYPE type)
+{
+ if (verbose) amrex::Print() << "\tmParticle::ForceSpreading\n";
+ const auto& gm = mContainer->GetParGDB()->Geom(LOCAL_LEVEL);
+ auto plo = gm.ProbLoArray();
+ auto dxi = gm.CellSizeArray();
+ for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
+ const Long np = pti.numParticles();
+ const auto& particles = pti.GetArrayOfStructs();
+ auto Uarray = EulerForce[pti].array();
+ auto& attri = pti.GetAttribs();
+
+ auto *const fxP_ptr = attri[P_ATTR::Fx_Marker].data();
+ auto *const fyP_ptr = attri[P_ATTR::Fy_Marker].data();
+ auto *const fzP_ptr = attri[P_ATTR::Fz_Marker].data();
+ auto *const mxP_ptr = attri[P_ATTR::Mx_Marker].data();
+ auto *const myP_ptr = attri[P_ATTR::My_Marker].data();
+ auto *const mzP_ptr = attri[P_ATTR::Mz_Marker].data();
+ const auto *const p_ptr = particles().data();
+
+ auto loc_ptr = kernel.location;
+ auto dv = kernel.dv;
+ auto force_index = ParticleProperties::euler_force_index;
+ amrex::ParallelFor(np, [=]
+ AMREX_GPU_DEVICE (int i) noexcept{
+ ForceSpreading_cic(p_ptr[i], loc_ptr[0], loc_ptr[1], loc_ptr[2],
+ fxP_ptr[i], fyP_ptr[i], fzP_ptr[i],
+ mxP_ptr[i], myP_ptr[i], mzP_ptr[i],
+ Uarray, force_index, dv, plo, dxi, type);
+ });
+ }
+ //barrier for sync;
+ amrex::ParallelDescriptor::Barrier();
+
+ using pc = mParticleContainer::SuperParticleType;
+ // Each Processor
+ auto fx = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fx_Marker);});
+ auto fy = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fy_Marker);});
+ auto fz = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Fz_Marker);});
+ auto mx = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Mx_Marker);});
+ auto my = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::My_Marker);});
+ auto mz = amrex::ReduceSum( *mContainer, [=]AMREX_GPU_HOST_DEVICE(const pc& p)->ParticleReal{return p.rdata(P_ATTR::Mz_Marker);});
+ // MPI sum reduce -> current particle all IB force and moment
+ amrex::ParallelAllReduce::Sum(fx, ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(fy, ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(fz, ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(mx, ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(my, ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(mz, ParallelDescriptor::Communicator());
+
+ kernel.ib_force = {fx, fy, fz};
+ kernel.ib_moment = {mx, my, mz};
+
+ EulerForce.SumBoundary(ParticleProperties::euler_force_index, 3, gm.periodicity());
+
+ // if (false) {
+ // // Check the Multifab
+ // // Open a file stream for output
+ // std::ofstream outFile("EulerForce.txt");
+
+ // // Check the Multifab
+ // // for (MFIter mfi(EulerForce, TilingIfNotGPU()); mfi.isValid(); ++mfi)
+ // for (MFIter mfi(EulerForce, TilingIfNotGPU()); mfi.isValid(); ++mfi)
+ // {
+ // const Box& bx = mfi.validbox();
+ // outFile << "Box: " << bx << "\n"
+ // << "From: (" << bx.smallEnd(0) << ", " << bx.smallEnd(1) << ", " << bx.smallEnd(2) << ") "
+ // << "To: (" << bx.bigEnd(0) << ", " << bx.bigEnd(1) << ", " << bx.bigEnd(2) << ")\n";
+
+ // Array4 const& a = EulerForce[mfi].array();
+
+ // // CPU context or illustrative purposes only
+ // for (int k = bx.smallEnd(2); k <= bx.bigEnd(2); ++k) {
+ // for (int j = bx.smallEnd(1); j <= bx.bigEnd(1); ++j) {
+ // for (int i = bx.smallEnd(0); i <= bx.bigEnd(0); ++i) {
+ // // This print statement is for demonstration and should not be used in actual GPU code.
+ // outFile << "Processing i: " << i << ", j: " << j << ", k: " << k << " " << a(i,j,k,0) << " " << a(i,j,k,1) << " " << a(i,j,k,2) << "\n";
+ // }
+ // }
+ // }
+ // }
+
+ // // Close the file when done
+ // outFile.close();
+ // }
+
+}
+
+void mParticle::ResetLargrangianPoints()
+{
+ if (verbose) amrex::Print() << "\tmParticle::ResetLargrangianPoints\n";
+
+ for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
+ const Long np = pti.numParticles();
+ auto& attri = pti.GetAttribs();
+
+ auto *const vUP_ptr = attri[P_ATTR::U_Marker].data();
+ auto *const vVP_ptr = attri[P_ATTR::V_Marker].data();
+ auto *const vWP_ptr = attri[P_ATTR::W_Marker].data();
+ auto *const fxP_ptr = attri[P_ATTR::Fx_Marker].data();
+ auto *const fyP_ptr = attri[P_ATTR::Fy_Marker].data();
+ auto *const fzP_ptr = attri[P_ATTR::Fz_Marker].data();
+ auto *const mxP_ptr = attri[P_ATTR::Mx_Marker].data();
+ auto *const myP_ptr = attri[P_ATTR::My_Marker].data();
+ auto *const mzP_ptr = attri[P_ATTR::Mz_Marker].data();
+ amrex::ParallelFor(np, [=]
+ AMREX_GPU_DEVICE (int i) noexcept{
+ vUP_ptr[i] = 0.0;
+ vVP_ptr[i] = 0.0;
+ vWP_ptr[i] = 0.0;
+ fxP_ptr[i] = 0.0;
+ fyP_ptr[i] = 0.0;
+ fzP_ptr[i] = 0.0;
+ mxP_ptr[i] = 0.0;
+ myP_ptr[i] = 0.0;
+ mzP_ptr[i] = 0.0;
+ });
+ }
+}
+
+void mParticle::UpdateParticles(int iStep,
+ Real time,
+ const MultiFab& Euler_old,
+ const MultiFab& Euler,
+ MultiFab& phi_nodal,
+ MultiFab& pvf,
+ Real dt)
+{
+ if (verbose) amrex::Print() << "mParticle::UpdateParticles\n";
+ // start record
+ auto UpdateParticlesStart = ParallelDescriptor::second();
+
+ //Particle Collision calculation
+ DoParticleCollision(ParticleProperties::collision_model);
+
+ MultiFab AllParticlePVF(pvf.boxArray(), pvf.DistributionMap(), pvf.nComp(), pvf.nGrow());
+ AllParticlePVF.setVal(0.0);
+
+ //continue condition 6DOF
+ for(auto& kernel : particle_kernels){
+
+ calculate_phi_nodal(phi_nodal, kernel);
+ nodal_phi_to_pvf(pvf, phi_nodal);
+
+ // // fixed particle
+ // if( ( kernel.TL.sum() == 0 ) &&
+ // ( kernel.RL.sum() == 0 ) ) {
+ // amrex::Print() << "Particle (" << kernel.id << ") is fixed\n";
+ // MultiFab::Add(AllParticlePVF, pvf, 0, 0, 1, 0); // do not copy ghost cell values
+ // continue;
+ // }
+
+ int ncomp = pvf.nComp();
+ int ngrow = pvf.nGrow();
+ MultiFab pvf_old(pvf.boxArray(), pvf.DistributionMap(), ncomp, ngrow);
+ MultiFab::Copy(pvf_old, pvf, 0, 0, ncomp, ngrow);
+
+ // bool at_least_one_free_trans_motion = ( kernel.TL[0] == 2 ) ||
+ // ( kernel.TL[1] == 2 ) ||
+ // ( kernel.TL[2] == 2 );
+ // bool at_least_one_free_rot_motion = ( kernel.RL[0] == 2 ) ||
+ // ( kernel.RL[1] == 2 ) ||
+ // ( kernel.RL[2] == 2 );
+
+ int loop = ParticleProperties::loop_solid;
+
+ while (loop > 0 && iStep > ParticleProperties::start_step) {
+
+ // if(at_least_one_free_trans_motion) {
+ kernel.sum_u_new.scale(0.0);
+ kernel.sum_u_old.scale(0.0);
+ // sum U
+ CalculateSumU_cir(kernel.sum_u_new, Euler, pvf, ParticleProperties::euler_velocity_index);
+ CalculateSumU_cir(kernel.sum_u_old, Euler_old, pvf_old, ParticleProperties::euler_velocity_index);
+ amrex::ParallelAllReduce::Sum(kernel.sum_u_new.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(kernel.sum_u_old.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
+ // }
+
+ // if(at_least_one_free_rot_motion) {
+ kernel.sum_t_new.scale(0.0);
+ kernel.sum_t_old.scale(0.0);
+ // sum T
+ CalculateSumT_cir(kernel.sum_t_new, Euler, pvf, kernel.location, ParticleProperties::euler_velocity_index);
+ CalculateSumT_cir(kernel.sum_t_old, Euler_old, pvf_old, kernel.location, ParticleProperties::euler_velocity_index);
+ amrex::ParallelAllReduce::Sum(kernel.sum_t_new.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
+ amrex::ParallelAllReduce::Sum(kernel.sum_t_old.dataPtr(), 3, amrex::ParallelDescriptor::Communicator());
+ // }
+
+ // 6DOF
+ if(ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber()){
+
+ for(auto idir : {0,1,2})
+ {
+ //TL
+ if (kernel.TL[idir] == 0) {
+ kernel.velocity[idir] = 0.0;
+ }
+ else if (kernel.TL[idir] == 1) {
+ kernel.location[idir] = kernel.location_old[idir] + (kernel.velocity[idir] + kernel.velocity_old[idir]) * dt * 0.5;
+ }
+ else if (kernel.TL[idir] == 2) {
+ if(!ParticleProperties::Uhlmann){
+ kernel.velocity[idir] = kernel.velocity_old[idir]
+ + ((kernel.sum_u_new[idir] - kernel.sum_u_old[idir]) * ParticleProperties::euler_fluid_rho / dt
+ - kernel.ib_force[idir] * ParticleProperties::euler_fluid_rho
+ + m_gravity[idir] * (kernel.rho - ParticleProperties::euler_fluid_rho) * kernel.Vp
+ + kernel.Fcp[idir]) * dt / kernel.rho / kernel.Vp ;
+ }else{
+ //Uhlmann
+ kernel.velocity[idir] = kernel.velocity_old[idir]
+ + (ParticleProperties::euler_fluid_rho / kernel.Vp /(ParticleProperties::euler_fluid_rho - kernel.rho)*kernel.ib_force[idir]
+ + m_gravity[idir]) * dt;
+ }
+ kernel.location[idir] = kernel.location_old[idir] + (kernel.velocity[idir] + kernel.velocity_old[idir]) * dt * 0.5;
+ }
+ else {
+ amrex::Print() << "Particle (" << kernel.id << ") has wrong TL"<< direction_str[idir] <<" value\n";
+ amrex::Abort("Stop here!");
+ }
+ //RL
+ if (kernel.RL[idir] == 0) {
+ kernel.omega[idir] = 0.0;
+ }
+ else if (kernel.RL[idir] == 1) {
+ }
+ else if (kernel.RL[idir] == 2) {
+ if(!ParticleProperties::Uhlmann){
+ kernel.omega[idir] = kernel.omega_old[idir]
+ + ((kernel.sum_t_new[idir] - kernel.sum_t_old[idir]) * ParticleProperties::euler_fluid_rho / dt
+ - kernel.ib_moment[idir] * ParticleProperties::euler_fluid_rho
+ + kernel.Tcp[idir]) * dt / cal_momentum(kernel.rho, kernel.radius);
+ }else{
+ //Uhlmann
+ kernel.omega[idir] = kernel.omega_old[idir]
+ + ParticleProperties::euler_fluid_rho /(ParticleProperties::euler_fluid_rho - kernel.rho) * kernel.ib_moment[idir] * kernel.dv
+ / cal_momentum(kernel.rho, kernel.radius) * kernel.rho * dt;
+ }
+ }
+ else {
+ amrex::Print() << "Particle (" << kernel.id << ") has wrong RL"<< direction_str[idir] <<" value\n";
+ amrex::Abort("Stop here!");
+ }
+
+ }
+ }
+ ParallelDescriptor::Bcast(&kernel.location[0],3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.location_old[0],3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.velocity[0],3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.velocity_old[0],3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.omega[0],3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.omega_old[0],3,ParallelDescriptor::IOProcessorNumber());
+
+ loop--;
+
+ if (loop > 0) {
+ calculate_phi_nodal(phi_nodal, kernel);
+ nodal_phi_to_pvf(pvf, phi_nodal);
+ }
+
+ }
+
+ RecordOldValue(kernel);
+ MultiFab::Add(AllParticlePVF, pvf, 0, 0, 1, 0); // do not copy ghost cell values
+ }
+ // calculate the pvf based on the information of all particles
+ MultiFab::Copy(pvf, AllParticlePVF, 0, 0, 1, pvf.nGrow());
+
+ spend_time += ParallelDescriptor::second() - UpdateParticlesStart;
+ ParallelDescriptor::ReduceRealMax(spend_time);
+ amrex::Print() << "[DIBM] IB and update particle, step : "<< iStep <<", time : " << spend_time << "\n";
+
+ int particle_write_freq = ParticleProperties::write_freq;
+ if (iStep % particle_write_freq == 0) {
+ for(auto kernel: particle_kernels)
+ WriteIBForceAndMoment(iStep, time, dt, kernel);
+ }
+
+ if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 4));
+}
+
+void mParticle::DoParticleCollision(int model)
+{
+ if(particle_kernels.size() < 2 ) return ;
+
+ if (verbose) amrex::Print() << "\tmParticle::DoParticleCollision\n";
+
+ if(ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber()){
+ for(const auto& kernel : particle_kernels){
+ m_Collision.InsertParticle(kernel.location, kernel.velocity, kernel.radius, kernel.rho);
+ }
+
+ m_Collision.takeModel(model);
+
+ for(auto & particle_kernel : particle_kernels){
+ particle_kernel.Fcp = m_Collision.Particles.front().preForece
+ * particle_kernel.Vp * particle_kernel.rho * m_gravity.vectorLength();
+ m_Collision.Particles.pop_front();
+ }
+ }
+ for(auto& kernel : particle_kernels){
+ ParallelDescriptor::Bcast(kernel.Fcp.dataPtr(), 3, ParallelDescriptor::IOProcessorNumber());
+ }
+}
+
+void mParticle::ComputeLagrangianForce(Real dt,
+ const kernel& kernel)
+{
+
+ if (verbose) amrex::Print() << "\tmParticle::ComputeLagrangianForce\n";
+
+ Real Ub = kernel.velocity[0];
+ Real Vb = kernel.velocity[1];
+ Real Wb = kernel.velocity[2];
+ Real Px = kernel.location[0];
+ Real Py = kernel.location[1];
+ Real Pz = kernel.location[2];
+
+ for(mParIter pti(*mContainer, LOCAL_LEVEL); pti.isValid(); ++pti){
+ const Long np = pti.numParticles();
+ auto& attri = pti.GetAttribs();
+ auto const* p_ptr = pti.GetArrayOfStructs().data();
+
+ auto* Up = attri[P_ATTR::U_Marker].data();
+ auto* Vp = attri[P_ATTR::V_Marker].data();
+ auto* Wp = attri[P_ATTR::W_Marker].data();
+ auto *FxP = attri[P_ATTR::Fx_Marker].data();
+ auto *FyP = attri[P_ATTR::Fy_Marker].data();
+ auto *FzP = attri[P_ATTR::Fz_Marker].data();
+
+ amrex::ParallelFor(np,
+ [=] AMREX_GPU_DEVICE (int i) noexcept{
+ auto Ur = (kernel.omega).crossProduct(RealVect(p_ptr[i].pos(0) - Px, p_ptr[i].pos(1) - Py, p_ptr[i].pos(2) - Pz));
+ FxP[i] = (Ub + Ur[0] - Up[i])/dt; //
+ FyP[i] = (Vb + Ur[1] - Vp[i])/dt; //
+ FzP[i] = (Wb + Ur[2] - Wp[i])/dt; //
+ });
+ }
+ if (verbose) mContainer->WriteAsciiFile(amrex::Concatenate("particle", 3));
+}
+
+void mParticle::VelocityCorrection(amrex::MultiFab &Euler, amrex::MultiFab &EulerForce, Real dt) const
+{
+ if(verbose) amrex::Print() << "\tmParticle::VelocityCorrection\n";
+ MultiFab::Saxpy(Euler, dt, EulerForce, ParticleProperties::euler_force_index, ParticleProperties::euler_velocity_index, 3, 0); //VelocityCorrection
+}
+
+void mParticle::RecordOldValue(kernel& kernel)
+{
+ kernel.location_old = kernel.location;
+ kernel.velocity_old = kernel.velocity;
+ kernel.omega_old = kernel.omega;
+}
+
+void mParticle::WriteParticleFile(int index)
+{
+ mContainer->WriteAsciiFile(amrex::Concatenate("particle", index));
+}
+
+void mParticle::WriteIBForceAndMoment(int step, amrex::Real time, amrex::Real dt, kernel& current_kernel)
+{
+
+ if(amrex::ParallelDescriptor::MyProc() != ParallelDescriptor::IOProcessorNumber()) return;
+
+ std::string file("IB_Particle_" + std::to_string(current_kernel.id) + ".csv");
+ std::ofstream out_ib_force;
+
+ std::string head;
+ if(!fs::exists(file)){
+ head = "iStep,time,X,Y,Z,Vx,Vy,Vz,Rx,Ry,Rz,Fx,Fy,Fz,Mx,My,Mz,Fcpx,Fcpy,Fcpz,Tcpx,Tcpy,Tcpz,SumUx,SumUy,SumUz,SumTx,SumTy,SumTz\n";
+ }else{
+ head = "";
+ }
+
+ out_ib_force.open(file, std::ios::app);
+ if(!out_ib_force.is_open()){
+ amrex::Print() << "[Particle] write particle file error , step: " << step;
+ }else{
+ out_ib_force << head << step << "," << time << ","
+ << current_kernel.location[0] << "," << current_kernel.location[1] << "," << current_kernel.location[2] << ","
+ << current_kernel.velocity[0] << "," << current_kernel.velocity[1] << "," << current_kernel.velocity[2] << ","
+ << current_kernel.omega[0] << "," << current_kernel.omega[1] << "," << current_kernel.omega[2] << ","
+ << current_kernel.ib_force[0] << "," << current_kernel.ib_force[1] << "," << current_kernel.ib_force[2] << ","
+ << current_kernel.ib_moment[0] << "," << current_kernel.ib_moment[1] << "," << current_kernel.ib_moment[2] << ","
+ << current_kernel.Fcp[0] << "," << current_kernel.Fcp[1] << "," << current_kernel.Fcp[2] << ","
+ << current_kernel.Tcp[0] << "," << current_kernel.Tcp[1] << "," << current_kernel.Tcp[2] << ","
+ << (current_kernel.sum_u_new[0] - current_kernel.sum_u_old[0])/dt << ","
+ << (current_kernel.sum_u_new[1] - current_kernel.sum_u_old[1])/dt << ","
+ << (current_kernel.sum_u_new[2] - current_kernel.sum_u_old[2])/dt << ","
+ << (current_kernel.sum_t_new[0] - current_kernel.sum_t_old[0])/dt << ","
+ << (current_kernel.sum_t_new[1] - current_kernel.sum_t_old[1])/dt << ","
+ << (current_kernel.sum_t_new[2] - current_kernel.sum_t_old[2])/dt << "\n";
+ }
+ out_ib_force.close();
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* Particles member function */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+void Particles::create_particles(const Geometry &gm,
+ const DistributionMapping & dm,
+ const BoxArray & ba)
+{
+ amrex::Print() << "[Particle] : create Particle Container\n";
+ if(particle->mContainer != nullptr){
+ delete particle->mContainer;
+ particle->mContainer = nullptr;
+ }
+ particle->mContainer = new mParticleContainer(gm, dm, ba);
+
+ //get particle tile
+ std::pair key{0,0};
+ auto& particleTileTmp = particle->mContainer->GetParticles(0)[key];
+ //insert markers
+ if ( ParallelDescriptor::MyProc() == ParallelDescriptor::IOProcessorNumber() ) {
+ //insert particle's markers
+ //Real phiK = 0;
+ for(int marker_index = 0; marker_index < particle->particle_kernels[0].ml; marker_index++){
+ //insert code
+ mParticleContainer::ParticleType markerP;
+ markerP.id() = marker_index + 1;
+ markerP.cpu() = ParallelDescriptor::MyProc();
+ markerP.pos(0) = particle->particle_kernels[0].location[0];
+ markerP.pos(1) = particle->particle_kernels[0].location[1];
+ markerP.pos(2) = particle->particle_kernels[0].location[2];
+
+ std::array Marker_attr;
+ Marker_attr[U_Marker] = 0.0;
+ Marker_attr[V_Marker] = 0.0;
+ Marker_attr[W_Marker] = 0.0;
+ Marker_attr[Fx_Marker] = 0.0;
+ Marker_attr[Fy_Marker] = 0.0;
+ Marker_attr[Fz_Marker] = 0.0;
+
+ particleTileTmp.push_back(markerP);
+ particleTileTmp.push_back_real(Marker_attr);
+ }
+ }
+ particle->mContainer->Redistribute(); // Still needs to redistribute here!
+
+ ParticleProperties::plo = gm.ProbLoArray();
+ ParticleProperties::phi = gm.ProbHiArray();
+ ParticleProperties::dx = gm.CellSizeArray();
+}
+
+mParticle* Particles::get_particles()
+{
+ return particle;
+}
+
+
+void Particles::init_particle(Real gravity, Real h)
+{
+ amrex::Print() << "[Particle] : create Particle's kernel\n";
+ particle = new mParticle;
+ if(particle != nullptr){
+ isInitial = true;
+ particle->InitParticles(
+ ParticleProperties::_x,
+ ParticleProperties::_y,
+ ParticleProperties::_z,
+ ParticleProperties::_rho,
+ ParticleProperties::Vx,
+ ParticleProperties::Vy,
+ ParticleProperties::Vz,
+ ParticleProperties::Ox,
+ ParticleProperties::Oy,
+ ParticleProperties::Oz,
+ ParticleProperties::TLX,
+ ParticleProperties::TLY,
+ ParticleProperties::TLZ,
+ ParticleProperties::RLX,
+ ParticleProperties::RLY,
+ ParticleProperties::RLZ,
+ ParticleProperties::_radius,
+ h,
+ gravity,
+ ParticleProperties::verbose);
+ }
+
+}
+
+void Particles::Restart(Real gravity, Real h, int iStep)
+{
+ amrex::Print() << "[Particle] : restart Particle's kernel, step :" << iStep << "\n"
+ << "\tstart read particle csv file , default name is IB_Particle_x.csv\n"
+ << "\tdo not delete those file before \"restart\"\n\n";
+ delete particle;
+ particle = new mParticle;
+ particle->InitParticles(
+ ParticleProperties::_x,
+ ParticleProperties::_y,
+ ParticleProperties::_z,
+ ParticleProperties::_rho,
+ ParticleProperties::Vx,
+ ParticleProperties::Vy,
+ ParticleProperties::Vz,
+ ParticleProperties::Ox,
+ ParticleProperties::Oy,
+ ParticleProperties::Oz,
+ ParticleProperties::TLX,
+ ParticleProperties::TLY,
+ ParticleProperties::TLZ,
+ ParticleProperties::RLX,
+ ParticleProperties::RLY,
+ ParticleProperties::RLZ,
+ ParticleProperties::_radius,
+ h,
+ gravity,
+ ParticleProperties::verbose);
+ //deal in IO processor
+ //start read csv file
+ for(auto& kernel : particle->particle_kernels){
+ //filename
+ if(amrex::ParallelDescriptor::MyProc() == amrex::ParallelDescriptor::IOProcessorNumber()){
+ std::string fileName = "IB_Particle_" + std::to_string(kernel.id) + ".csv";
+ std::string tmpfile = "tmp" + fileName;
+ //file stream
+ std::ifstream particle_data(fileName);
+ std::ofstream particle_file(tmpfile);
+ // open state
+ if(!particle_data.is_open() || !particle_file.is_open()){
+ amrex::Abort("\tCan not open particle file : " + fileName);
+ }
+ std::string lineData;
+ int line{0};
+ while(std::getline(particle_data, lineData)){
+ line++;
+ particle_file << lineData << "\n";
+ if(line <= iStep) {
+ continue;
+ }
+ //old location
+ //iStep,time,X,Y,Z,Vx,Vy,Vz,Rx,Ry,Rz,Fx,Fy,Fz,Mx,My,Mz,Fcpx,Fcpy,Fcpz,Tcpx,Tcpy,Tcpz
+ if(line == iStep + 1){
+ std::stringstream ss(lineData);
+ std::string data;
+ std::vector dataStruct;
+ while(std::getline(ss, data, ',')){
+ dataStruct.emplace_back(std::stod(data));
+ }
+ kernel.location[0] = dataStruct[2];
+ kernel.location[1] = dataStruct[3];
+ kernel.location[2] = dataStruct[4];
+ kernel.velocity[0] = dataStruct[5];
+ kernel.velocity[1] = dataStruct[6];
+ kernel.velocity[2] = dataStruct[7];
+ kernel.omega[0] = dataStruct[8];
+ kernel.omega[1] = dataStruct[9];
+ kernel.omega[2] = dataStruct[10];
+
+ kernel.location_old = kernel.location;
+ kernel.velocity_old = kernel.velocity;
+ kernel.omega_old = kernel.omega;
+ break;
+ }
+ else
+ break;
+ }
+ particle_data.close();
+ particle_file.close();
+ std::remove(fileName.c_str());
+ std::rename(tmpfile.c_str(), fileName.c_str());
+ }
+ ParallelDescriptor::Bcast(&kernel.location[0], 3, ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.velocity[0], 3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.omega[0], 3,ParallelDescriptor::IOProcessorNumber());
+
+ ParallelDescriptor::Bcast(&kernel.location_old[0], 3, ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.velocity_old[0], 3,ParallelDescriptor::IOProcessorNumber());
+ ParallelDescriptor::Bcast(&kernel.omega_old[0], 3,ParallelDescriptor::IOProcessorNumber());
+ }
+
+ isInitial = true;
+}
+
+void Particles::Initialize()
+{
+ ParmParse pp("particle");
+
+ std::string particle_inputfile;
+ std::string particle_init_file;
+ pp.get("input",particle_inputfile);
+
+ if(!particle_inputfile.empty()){
+ ParmParse p_file(particle_inputfile);
+ p_file.query("init", particle_init_file);
+ p_file.getarr("x", ParticleProperties::_x);
+ p_file.getarr("y", ParticleProperties::_y);
+ p_file.getarr("z", ParticleProperties::_z);
+ p_file.getarr("rho", ParticleProperties::_rho);
+ p_file.getarr("velocity_x", ParticleProperties::Vx);
+ p_file.getarr("velocity_y", ParticleProperties::Vy);
+ p_file.getarr("velocity_z", ParticleProperties::Vz);
+ p_file.getarr("omega_x", ParticleProperties::Ox);
+ p_file.getarr("omega_y", ParticleProperties::Oy);
+ p_file.getarr("omega_z", ParticleProperties::Oz);
+ p_file.getarr("TLX", ParticleProperties::TLX);
+ p_file.getarr("TLY", ParticleProperties::TLY);
+ p_file.getarr("TLZ", ParticleProperties::TLZ);
+ p_file.getarr("RLX", ParticleProperties::RLX);
+ p_file.getarr("RLY", ParticleProperties::RLY);
+ p_file.getarr("RLZ", ParticleProperties::RLZ);
+ p_file.getarr("radius", ParticleProperties::_radius);
+ p_file.query("RD", ParticleProperties::rd);
+ p_file.query("LOOP_NS", ParticleProperties::loop_ns);
+ p_file.query("LOOP_SOLID", ParticleProperties::loop_solid);
+ p_file.query("verbose", ParticleProperties::verbose);
+ p_file.query("start_step", ParticleProperties::start_step);
+ p_file.query("Uhlmann", ParticleProperties::Uhlmann);
+ p_file.query("collision_model", ParticleProperties::collision_model);
+ p_file.query("write_freq", ParticleProperties::write_freq);
+
+ ParmParse ns("ns");
+ ns.get("fluid_rho", ParticleProperties::euler_fluid_rho);
+
+ ParmParse level_parse("amr");
+ level_parse.get("max_level", ParticleProperties::euler_finest_level);
+
+ ParmParse geometry_parse("geometry");
+ geometry_parse.getarr("prob_lo", ParticleProperties::GLO);
+ geometry_parse.getarr("prob_hi", ParticleProperties::GHI);
+ amrex::Print() << "[Particle] : Reading partilces cfg file : " << particle_inputfile << "\n"
+ << " Particle's level : " << ParticleProperties::euler_finest_level << "\n";
+
+ if(!particle_init_file.empty()){
+ ParticleProperties::init_particle_from_file = true;
+ //clear particle position container
+ ParticleProperties::_x.clear();
+ ParticleProperties::_y.clear();
+ ParticleProperties::_z.clear();
+ // parse particle's location data
+ std::ifstream init_particle(particle_init_file);
+ std::string line_data;
+ while(std::getline(init_particle, line_data)){
+ // id x_location y_location z_location
+ std::istringstream line(line_data);
+ std::vector str_tokne;
+ std::string token;
+ while(line >> token){
+ str_tokne.push_back(token);
+ }
+
+ ParticleProperties::_x.push_back(std::stod(str_tokne[0]));
+ ParticleProperties::_y.push_back(std::stod(str_tokne[1]));
+ ParticleProperties::_z.push_back(std::stod(str_tokne[2]));
+ }
+ ParticleProperties::_x.shrink_to_fit();
+ ParticleProperties::_y.shrink_to_fit();
+ ParticleProperties::_z.shrink_to_fit();
+ amrex::Print() << " initial Particle by file : " << particle_init_file
+ << " particle's size : " << ParticleProperties::_x.size() << "\n";
+ }
+
+ }else {
+ amrex::Abort("[Particle] : can't read particles settings, pls check your config file \"particle.input\"");
+ }
+}
+
+int Particles::ParticleFinestLevel()
+{
+ return ParticleProperties::euler_finest_level;
+}
diff --git a/Source/Diffusion.H b/Source/Diffusion.H
index 58726ef6..8c65b91d 100644
--- a/Source/Diffusion.H
+++ b/Source/Diffusion.H
@@ -1,268 +1,273 @@
-
-#ifndef IAMR_Diffusion_H_
-#define IAMR_Diffusion_H_
-
-
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-
-#include
-#ifdef AMREX_USE_EB
-#include
-#include
-#else
-#include
-#include
-#endif
-
-
-//
-// Useful enumeration of the different forms of the diffusion terms
-//
-enum DiffusionForm { RhoInverse_Laplacian_S, Laplacian_SoverRho, Laplacian_S };
-
-class NavierStokesBase;
-
-class Diffusion
-{
-public:
-
- Diffusion ();
-
- Diffusion (amrex::Amr* Parent,
- NavierStokesBase* Caller,
- Diffusion* coarser,
- int num_state,
- amrex::FluxRegister* Viscflux_reg,
- const amrex::Vector& _is_diffusive);
-
- void echo_settings () const;
-
- amrex::FluxRegister* viscFluxReg ();
-
- static amrex::Real get_scaled_abs_tol (const amrex::MultiFab& rhs,
- amrex::Real reduction);
-
- void diffuse_scalar (const amrex::Vector& S_old,
- const amrex::Vector& Rho_old,
- amrex::Vector& S_new,
- const amrex::Vector& Rho_new,
- int S_comp,
- int nComp,
- int Rho_comp,
- amrex::Real prev_time,
- amrex::Real curr_time,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- amrex::MultiFab* const* fluxn,
- amrex::MultiFab* const* fluxnp1,
- int fluxComp,
- amrex::MultiFab* delta_rhs,
- int rhsComp,
- const amrex::MultiFab* alpha,
- int alpha_in_comp,
- const amrex::MultiFab* const* betan,
- const amrex::MultiFab* const* betanp1,
- int betaComp,
- const amrex::IntVect& cratio,
- const amrex::BCRec& bc,
- const amrex::Geometry& geom,
- bool add_old_time_divFlux = true,
- const amrex::Vector& is_diffusive = amrex::Vector());
-
- void diffuse_velocity (amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- amrex::MultiFab* delta_rhs,
- const amrex::MultiFab* const* betan,
- const amrex::MultiFab* betanCC,
- const amrex::MultiFab* const* betanp1,
- const amrex::MultiFab* betanp1CC);
-
- void diffuse_velocity (amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- amrex::MultiFab* delta_rhs,
- int rhsComp,
- const amrex::MultiFab* const* betan,
- const amrex::MultiFab* betanCC,
- const amrex::MultiFab* const* betanp1,
- const amrex::MultiFab* betanp1CC,
- int betaComp);
-
- void diffuse_tensor_velocity (amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- amrex::MultiFab* delta_rhs,
- int rhsComp,
- const amrex::MultiFab* const* betan,
- const amrex::MultiFab* betanCC,
- const amrex::MultiFab* const* betanp1,
- const amrex::MultiFab* betanp1CC,
- int betaComp);
-
- void diffuse_Vsync (amrex::MultiFab& Vsync,
- amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- const amrex::MultiFab* const* beta,
- int betaComp = 0,
- bool update_fluxreg = true);
-
- void diffuse_tensor_Vsync (amrex::MultiFab& Vsync,
- amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- const amrex::MultiFab* const* beta,
- int betaComp,
- bool update_fluxreg);
-
-
- void diffuse_Ssync (amrex::MultiFab& Ssync,
- int sigma,
- int nComp,
- amrex::Real dt,
- amrex::Real be_cn_theta,
- const amrex::MultiFab& rho_half,
- int rho_flag,
- amrex::MultiFab* const* flux,
- int fluxComp,
- const amrex::MultiFab* const* beta,
- int betaComp,
- const amrex::MultiFab* alpha,
- int alphaComp);
-
-
- void getViscTerms (amrex::MultiFab& visc_terms,
- int src_comp,
- int comp,
- amrex::Real time,
- int rho_flag,
- const amrex::MultiFab* const* beta,
- int betaComp);
-
-
- void getTensorViscTerms (amrex::MultiFab& visc_terms,
- amrex::Real time,
- const amrex::MultiFab* const* beta,
- const amrex::MultiFab* betaCC,
- int betaComp);
-
- void FillBoundary (amrex::BndryRegister& bdry,
- int state_ind,
- int dest_comp,
- int num_comp,
- amrex::Real time,
- int rho_flag);
-
- static void checkBeta (const amrex::MultiFab* const* beta,
- int& allthere,
- int& allnull);
-
- static void checkBeta (const amrex::MultiFab* const* beta,
- int& allthere);
-
- [[nodiscard]] static int maxOrder ();
- [[nodiscard]] static int tensorMaxOrder ();
-
- static int set_rho_flag (DiffusionForm compDiffusionType);
-
- static void setDomainBC (std::array& mlmg_lobc,
- std::array& mlmg_hibc,
- const amrex::BCRec& bc);
-
- static void computeAlpha (amrex::MultiFab& alpha,
- std::pair& scalars,
- amrex::Real a,
- amrex::Real b,
- amrex::Real* rhsscale,
- const amrex::MultiFab* alpha_in,
- int alpha_in_comp,
- int rho_flag,
- const amrex::MultiFab* rho,
- int rho_comp);
-
-#ifdef AMREX_USE_EB
- static void setBeta(amrex::MLEBABecLap& op,
- const amrex::MultiFab* const* beta,
- int betaComp,
- int nComp = 1);
-
- static void setViscosity(amrex::MLEBTensorOp& tensorop,
- const amrex::MultiFab* const* beta,
- int betaComp,
- const amrex::MultiFab& beta_cc);
-#else
- static void setBeta(amrex::MLABecLaplacian& op,
- const amrex::MultiFab* const* beta,
- int betaComp,
- int nComp = 1);
-
- static void setViscosity(amrex::MLTensorOp& tensorop,
- const amrex::MultiFab* const* beta,
- int betaComp);
-#endif
-
- static void computeExtensiveFluxes(amrex::MLMG& a_mg,
- amrex::MultiFab& Soln,
- amrex::MultiFab* const* flux,
- int fluxComp,
- int ncomp,
- amrex::MultiFab* area,
- amrex::Real fac );
-
-protected:
-
- void setDomainBC (std::array& mlmg_lobc,
- std::array& mlmg_hibc,
- int src_comp);
-
-
- static void Finalize ();
- //
- // Data Required by Derived Classes
- //
- amrex::Amr* parent;
- NavierStokesBase* navier_stokes;
- const amrex::BoxArray& grids;
- const amrex::DistributionMapping& dmap;
- const int level;
- //
- // Static data.
- //
- static int scale_abec;
- static amrex::Vector is_diffusive; // Does variable diffuse?
- static int verbose;
- static amrex::Real visc_tol;
-
-private:
- //
- // The data.
- //
- Diffusion* coarser;
- Diffusion* finer;
- int NUM_STATE;
- amrex::IntVect crse_ratio;
- amrex::FluxRegister* viscflux_reg;
- //
- // Static data.
- //
- static int do_reflux;
- static int max_order;
- static int tensor_max_order;
-};
-
-#endif
+/*
+ * SPDX-FileCopyrightText: 1997 - 2023 Berkeley Lab
+ *
+ * SPDX-License-Identifier: LicenseRef-OpenSource
+ */
+
+#ifndef IAMR_Diffusion_H_
+#define IAMR_Diffusion_H_
+
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#ifdef AMREX_USE_EB
+#include
+#include
+#else
+#include
+#include
+#endif
+
+
+//
+// Useful enumeration of the different forms of the diffusion terms
+//
+enum DiffusionForm { RhoInverse_Laplacian_S, Laplacian_SoverRho, Laplacian_S };
+
+class NavierStokesBase;
+
+class Diffusion
+{
+public:
+
+ Diffusion ();
+
+ Diffusion (amrex::Amr* Parent,
+ NavierStokesBase* Caller,
+ Diffusion* coarser,
+ int num_state,
+ amrex::FluxRegister* Viscflux_reg,
+ const amrex::Vector& _is_diffusive);
+
+ void echo_settings () const;
+
+ amrex::FluxRegister* viscFluxReg ();
+
+ static amrex::Real get_scaled_abs_tol (const amrex::MultiFab& rhs,
+ amrex::Real reduction);
+
+ void diffuse_scalar (const amrex::Vector