Moved 3D RBC stuff to its own project

.github/workflows/ci_pipeline.yml

@@ -94,6 +94,7 @@ jobs:
           - PinTSimE
           - RDC
           - Resilience
+          - RayleighBenard
           - SDC_showdown
           - Second_orderSDC
           - soft_failure

pySDC/projects/GPU/configs/base_config.py

@@ -8,7 +8,7 @@ def get_config(args):
     if name[:2] == 'GS':
         from pySDC.projects.GPU.configs.GS_configs import get_config as _get_config
     elif name[:5] == 'RBC3D':
-        from pySDC.projects.GPU.configs.RBC3D_configs import get_config as _get_config
+        from pySDC.projects.RayleighBenard.RBC3D_configs import get_config as _get_config
     elif name[:3] == 'RBC':
         from pySDC.projects.GPU.configs.RBC_configs import get_config as _get_config
     else:

pySDC/projects/GPU/etc/venv_jusuf/modules.sh

@@ -1,5 +1,5 @@
 module --force purge
-module load Stages/2024
+module load Stages/2025
 module load GCC
 module load ParaStationMPI
 # module load NCCL
@@ -9,6 +9,6 @@ module load Python
 # module load CuPy
 module load FFTW
 module load mpi4py
-module load FFmpeg/.6.0
+# module load FFmpeg/.6.0
 # module load SciPy-Stack
 module load texlive

pySDC/projects/GPU/etc/venv_jusuf/requirements.txt

@@ -3,3 +3,4 @@ numpy
 matplotlib>=3.0
 dill>=0.2.6
 scipy>=1.14
+tqdm

pySDC/projects/GPU/etc/venv_jusuf/setup.sh

@@ -12,7 +12,8 @@ python3 -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}"
 
 source "${ABSOLUTE_PATH}"/activate.sh
 
-FFTW_LIBRARY_DIR="/p/software/jusuf/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" python3 -m pip install -e /p/project/ccstma/baumann7/mpi4py-fft
+
+FFTW_LIBRARY_DIR="/p/software/jusuf/stages/2025/software/FFTW/3.3.10-GCC-13.3.0/lib64/" python3 -m pip install -e /p/project1/ccstma/baumann7/mpi4py-fft
 python3 -m pip install -e /p/project1/ccstma/baumann7/qmat
 python3 -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt
 python3 -m pip install -e /p/project1/ccstma/baumann7/pySDC/

pySDC/projects/GPU/configs/RBC3D_configs.py → pySDC/projects/RayleighBenard/RBC3D_configs.py

@@ -212,6 +212,15 @@ def get_description(self, *args, **kwargs):
         return desc
 
 
+class RBC3DM2K2(RBC3Dverification):
+
+    def get_description(self, *args, **kwargs):
+        desc = super().get_description(*args, **kwargs)
+        desc['level_params']['nsweeps'] = 2
+        desc['sweeper_params']['num_nodes'] = 2
+        return desc
+
+
 class RBC3DM3K4(RBC3Dverification):
 
     def get_description(self, *args, **kwargs):
@@ -256,9 +265,9 @@ def get_description(self, *args, res=-1, dt=-1, **kwargs):
         return desc
 
 
-class RBC3DG4R4Ra1e5(RBC3Dverification):
+class RBC3DG4R4SDC22Ra1e5(RBC3DM2K2):
     Tend = 200
-    dt = 6e-2
+    dt = 4e-2
     res = 32
     converged = 50
 

pySDC/projects/GPU/analysis_scripts/RBC3D_order.py → pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_order.py

@@ -2,15 +2,15 @@
 import pickle
 import numpy as np
 from pySDC.helpers.fieldsIO import FieldsIO
-from pySDC.projects.GPU.configs.base_config import get_config
+from pySDC.projects.RayleighBenard.RBC3D_configs import get_config
 from pySDC.implementations.problem_classes.RayleighBenard3D import RayleighBenard3D
 from mpi4py import MPI
 import matplotlib.pyplot as plt
 from pySDC.helpers.plot_helper import figsize_by_journal
-from pySDC.projects.GPU.analysis_scripts.RBC3D_plotting_utils import get_plotting_style, savefig
+from pySDC.projects.RayleighBenard.analysis_scripts.plotting_utils import get_plotting_style, savefig
 
 step_sizes = {
-    'RBC3DG4R4Ra1e5': [8e-2, 4e-2, 2e-2, 1e-2, 5e-3],
+    'RBC3DG4R4SDC22Ra1e5': [5e-3 * 2**i for i in range(8)],
     'RBC3DG4R4SDC23Ra1e5': [5e-3 * 2**i for i in range(8)],
     'RBC3DG4R4SDC34Ra1e5': [5e-3 * 2**i for i in range(8)],
     'RBC3DG4R4SDC44Ra1e5': [5e-3 * 2**i for i in range(8)],
@@ -74,7 +74,7 @@ def plot_error_all_components(args):  # pragma: no cover
 def compare_order(Ra):  # pragma: no cover
     fig, ax = plt.subplots(figsize=figsize_by_journal('Nature_CS', 1, 0.6))
     if Ra == 1e5:
-        names = ['RK', 'Euler', 'SDC23', 'SDC34', 'SDC44'][::-1]
+        names = ['RK', 'Euler', 'SDC22', 'SDC23', 'SDC34', 'SDC44'][::-1]
         configs = [f'RBC3DG4R4{me}Ra1e5' for me in names]
         paths = [f'./data/RBC3DG4R4{me}Ra1e5-res-1-order.pickle' for me in names]
 
@@ -92,7 +92,7 @@ def compare_order(Ra):  # pragma: no cover
         ax.loglog(dt, e, **get_plotting_style(config))
 
     for _dt in dt:
-        for i in [1, 3, 4]:
+        for i in [1, 2, 3, 4]:
             ax.text(_dt, _dt**i, i, fontweight='bold', fontsize=14, ha='center', va='center')
             ax.loglog(dt, dt**i, ls=':', color='black')
 
@@ -103,7 +103,7 @@ def compare_order(Ra):  # pragma: no cover
 
 
 def run(args, dt, Tend):
-    from pySDC.projects.GPU.run_experiment import run_experiment
+    from pySDC.projects.RayleighBenard.run_experiment import run_experiment
     from pySDC.core.errors import ConvergenceError
 
     args['mode'] = 'run'
@@ -116,7 +116,7 @@ def run(args, dt, Tend):
     prob = desc['problem_class'](**desc['problem_params'])
 
     ic_config_name = type(config).__name__
-    for name in ['RK', 'Euler', 'O3', 'O4', 'SDC23', 'SDC34', 'SDC44']:
+    for name in ['RK', 'Euler', 'O3', 'O4', 'SDC23', 'SDC34', 'SDC44', 'SDC22']:
         ic_config_name = ic_config_name.replace(name, 'SDC34')
 
     ic_config = get_config({**args, 'config': ic_config_name})
@@ -134,7 +134,7 @@ def run(args, dt, Tend):
 
 
 if __name__ == '__main__':
-    from pySDC.projects.GPU.run_experiment import parse_args
+    from pySDC.projects.RayleighBenard.run_experiment import parse_args
 
     args = parse_args()
 

pySDC/projects/GPU/analysis_scripts/RBC3D_spectrum.py → pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_spectrum.py

@@ -1,5 +1,9 @@
-from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import get_pySDC_data
-from pySDC.projects.GPU.analysis_scripts.RBC3D_plotting_utils import figsize_by_journal, get_plotting_style, savefig
+from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import get_pySDC_data
+from pySDC.projects.RayleighBenard.analysis_scripts.plotting_utils import (
+    figsize_by_journal,
+    get_plotting_style,
+    savefig,
+)
 import matplotlib.pyplot as plt
 
 

pySDC/projects/GPU/analysis_scripts/plot_Nu.py → pySDC/projects/RayleighBenard/analysis_scripts/plot_Nu.py

@@ -3,7 +3,7 @@
 import numpy as np
 from scipy import integrate
 from pySDC.helpers.plot_helper import figsize_by_journal, setup_mpl
-from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import get_pySDC_data
+from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import get_pySDC_data
 
 setup_mpl()
 
@@ -39,9 +39,9 @@ def plot_Nu(res, dts, config_name, ref, ax, title):  # pragma: no cover
         if any(error > 1e-2):
             deviates = min(t_i[error > 1e-2])
             ax.axvline(deviates, color=last_line.get_color(), ls=':')
-            print(f'{title} dt={dt} Nu={Nu_mean:.3f}+={Nu_std:.3f}, deviates more than 1% from t={deviates:.2f}')
+            print(f'{title} dt={dt:.4f} Nu={Nu_mean:.3f}+={Nu_std:.3f}, deviates more than 1% from t={deviates:.2f}')
         else:
-            print(f'{title} dt={dt} Nu={Nu_mean:.3f}+={Nu_std:.3f}')
+            print(f'{title} dt={dt:.4f} Nu={Nu_mean:.3f}+={Nu_std:.3f}')
         ax.legend(frameon=True, loc='upper left')
 
 
@@ -50,12 +50,13 @@ def plot_Nu_over_time_Ra1e5():  # pragma: no cover
 
     res = 32
 
-    ref_data = get_pySDC_data(res=res, dt=0.01, config_name='RBC3DG4R4Ra1e5')
+    ref_data = get_pySDC_data(res=res, dt=0.01, config_name='RBC3DG4R4SDC44Ra1e5')
 
     plot_Nu(32, [0.06, 0.04, 0.02], 'RBC3DG4R4SDC34Ra1e5', ref_data, Nu_axs[0], 'SDC34')
     plot_Nu(32, [0.06, 0.05, 0.02, 0.01], 'RBC3DG4R4SDC23Ra1e5', ref_data, Nu_axs[1], 'SDC23')
     plot_Nu(32, [0.05, 0.04, 0.02, 0.01, 0.005], 'RBC3DG4R4RKRa1e5', ref_data, Nu_axs[2], 'RK443')
     plot_Nu(32, [0.02, 0.01, 0.005], 'RBC3DG4R4EulerRa1e5', ref_data, Nu_axs[3], 'RK111')
+    plot_Nu(32, [0.04, 0.02], 'RBC3DG4R4SDC22Ra1e5', ref_data, Nu_axs[3], 'SDC22')
 
     Nu_axs[-1].set_xlabel('$t$')
     Nu_axs[-1].set_ylabel('$Nu$')

pySDC/projects/GPU/analysis_scripts/RBC3D_plotting_utils.py → pySDC/projects/RayleighBenard/analysis_scripts/plotting_utils.py

@@ -1,5 +1,6 @@
 from pySDC.helpers.plot_helper import figsize_by_journal, setup_mpl
 import warnings
+import os
 
 setup_mpl()
 
@@ -8,7 +9,12 @@ def get_plotting_style(config):  # pragma: no cover
 
     args = {'color': None, 'ls': None, 'marker': None, 'markersize': 6, 'label': None}
 
-    if config == 'RBC3DG4R4SDC23Ra1e5':
+    if config == 'RBC3DG4R4SDC22Ra1e5':
+        args['color'] = 'tab:brown'
+        args['ls'] = '-'
+        args['marker'] = '3'
+        args['label'] = 'SDC22'
+    elif config == 'RBC3DG4R4SDC23Ra1e5':
         args['color'] = 'tab:blue'
         args['ls'] = '-'
         args['marker'] = 'o'
@@ -18,7 +24,7 @@ def get_plotting_style(config):  # pragma: no cover
         args['ls'] = '-'
         args['marker'] = '<'
         args['label'] = 'SDC34'
-    elif config in ['RBC3DG4R4SDC44Ra1e5', 'RBC3DG4R4Ra1e5']:
+    elif config == 'RBC3DG4R4SDC44Ra1e5':
         args['color'] = 'tab:green'
         args['ls'] = '-'
         args['marker'] = 'x'
@@ -40,6 +46,8 @@ def get_plotting_style(config):  # pragma: no cover
 
 
 def savefig(fig, name, format='pdf', base_path='./plots', **kwargs):  # pragma: no cover
+    os.makedirs(base_path, exist_ok=True)
+
     path = f'{base_path}/{name}.{format}'
     fig.savefig(path, bbox_inches='tight', **kwargs)
     print(f'Saved figure {path!r}')

pySDC/projects/GPU/analysis_scripts/process_RBC3D_data.py → pySDC/projects/RayleighBenard/analysis_scripts/process_RBC3D_data.py

@@ -1,5 +1,5 @@
-from pySDC.projects.GPU.configs.base_config import get_config
-from pySDC.projects.GPU.run_experiment import parse_args
+from pySDC.projects.RayleighBenard.RBC3D_configs import get_config
+from pySDC.projects.RayleighBenard.run_experiment import parse_args
 from pySDC.helpers.fieldsIO import FieldsIO
 import matplotlib.pyplot as plt
 from tqdm import tqdm
@@ -9,10 +9,11 @@
 import os
 
 
-def process_RBC3D_data(base_path='./data/RBC_time_averaged', plot=True, args=None, config=None):
+def process_RBC3D_data(base_path='./data/processed', plot=True, args=None, config=None):
     # prepare problem instance
     args = args if args else parse_args()
     comm = MPI.COMM_WORLD
+    args['procs'] = [1, 1, comm.size]
     config = config if config else get_config(args)
     desc = config.get_description(**args)
     P = desc['problem_class'](
@@ -204,7 +205,7 @@ def process_RBC3D_data(base_path='./data/RBC_time_averaged', plot=True, args=Non
     return path
 
 
-def get_pySDC_data(res=-1, dt=-1, config_name='RBC3DG4', base_path='data/RBC_time_averaged'):
+def get_pySDC_data(res=-1, dt=-1, config_name='RBC3DG4', base_path='data/processed'):
     path = f'{base_path}/{config_name}-res{res}-dt{dt:.0e}.pickle'
     with open(path, 'rb') as file:
         data = pickle.load(file)

pySDC/projects/RayleighBenard/environment.yml

@@ -0,0 +1,18 @@
+---
+
+name: pySDC
+channels:
+  - conda-forge
+dependencies:
+  - numpy>=1.15.4
+  - scipy>=0.17.1
+  - matplotlib>=3.0
+  - dill>=0.2.6
+  - mpich
+  - mpi4py-fft>=2.0.2
+  - mpi4py>=3.0.0
+  - tqdm
+  - pip
+  - pip:
+    - qmat>=0.1.8
+    - pytest-isolate-mpi

pySDC/projects/RayleighBenard/run_experiment.py

@@ -0,0 +1,87 @@
+def parse_args():
+    import argparse
+
+    def cast_to_bool(me):
+        return False if me in ['False', '0', 0] else True
+
+    def str_to_procs(me):
+        procs = me.split('/')
+        assert len(procs) == 3
+        return [int(p) for p in procs]
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--useGPU', type=cast_to_bool, help='Toggle for GPUs', default=False)
+    parser.add_argument(
+        '--mode',
+        type=str,
+        help='Mode for this script',
+        default=None,
+        choices=['run', 'plot'],
+    )
+    parser.add_argument('--config', type=str, help='Configuration to load', default=None)
+    parser.add_argument('--restart_idx', type=int, help='Restart from file by index', default=0)
+    parser.add_argument('--procs', type=str_to_procs, help='Processes in steps/sweeper/space', default='1/1/1')
+    parser.add_argument('--res', type=int, help='Space resolution along first axis', default=-1)
+    parser.add_argument('--dt', type=float, help='(Starting) Step size', default=-1)
+    parser.add_argument(
+        '--logger_level', type=int, help='Logger level on the first rank in space and in the sweeper', default=15
+    )
+    parser.add_argument('-o', type=str, help='output path', default='./')
+
+    return vars(parser.parse_args())
+
+
+def run_experiment(args, config, **kwargs):
+    import pickle
+    import os
+
+    from pySDC.implementations.controller_classes.controller_nonMPI import controller_nonMPI
+    from pySDC.helpers.stats_helper import filter_stats
+
+    type(config).base_path = args['o']
+    os.makedirs(f'{args["o"]}/data', exist_ok=True)
+
+    description = config.get_description(
+        useGPU=args['useGPU'], MPIsweeper=args['procs'][1] > 1, res=args['res'], dt=args['dt'], **kwargs
+    )
+    controller_params = config.get_controller_params(logger_level=args['logger_level'])
+
+    if args['useGPU']:
+        from pySDC.implementations.hooks.log_timings import GPUTimings
+
+        controller_params['hook_class'].append(GPUTimings)
+
+    assert (
+        config.comms[0].size == 1
+    ), 'Have not figured out how to do MPI controller with GPUs yet because I need NCCL for that!'
+    controller = controller_nonMPI(num_procs=1, controller_params=controller_params, description=description)
+    prob = controller.MS[0].levels[0].prob
+
+    u0, t0 = config.get_initial_condition(prob, restart_idx=args['restart_idx'])
+
+    config.prepare_caches(prob)
+
+    uend, stats = controller.run(u0=u0, t0=t0, Tend=config.Tend)
+
+    combined_stats = filter_stats(stats, comm=config.comm_world)
+
+    if config.comm_world.rank == config.comm_world.size - 1:
+        path = f'{config.base_path}/data/{config.get_path()}-stats-whole-run.pickle'
+        with open(path, 'wb') as file:
+            pickle.dump(combined_stats, file)
+        print(f'Stored stats in {path}', flush=True)
+
+    return uend
+
+
+if __name__ == '__main__':
+    from pySDC.projects.RayleighBenard.RBC3D_configs import get_config
+
+    args = parse_args()
+
+    config = get_config(args)
+
+    if args['mode'] == 'run':
+        run_experiment(args, config)
+    else:
+        raise NotImplementedError