diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index 2e9ee1b37b..c8249626af 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -94,6 +94,7 @@ jobs: - PinTSimE - RDC - Resilience + - RayleighBenard - SDC_showdown - Second_orderSDC - soft_failure diff --git a/pySDC/projects/GPU/configs/base_config.py b/pySDC/projects/GPU/configs/base_config.py index fe0754bc31..cebff83c31 100644 --- a/pySDC/projects/GPU/configs/base_config.py +++ b/pySDC/projects/GPU/configs/base_config.py @@ -8,7 +8,7 @@ def get_config(args): if name[:2] == 'GS': from pySDC.projects.GPU.configs.GS_configs import get_config as _get_config elif name[:5] == 'RBC3D': - from pySDC.projects.GPU.configs.RBC3D_configs import get_config as _get_config + from pySDC.projects.RayleighBenard.RBC3D_configs import get_config as _get_config elif name[:3] == 'RBC': from pySDC.projects.GPU.configs.RBC_configs import get_config as _get_config else: diff --git a/pySDC/projects/GPU/etc/venv_jusuf/modules.sh b/pySDC/projects/GPU/etc/venv_jusuf/modules.sh index 5d3e339756..8ebe9f2751 100644 --- a/pySDC/projects/GPU/etc/venv_jusuf/modules.sh +++ b/pySDC/projects/GPU/etc/venv_jusuf/modules.sh @@ -1,5 +1,5 @@ module --force purge -module load Stages/2024 +module load Stages/2025 module load GCC module load ParaStationMPI # module load NCCL @@ -9,6 +9,6 @@ module load Python # module load CuPy module load FFTW module load mpi4py -module load FFmpeg/.6.0 +# module load FFmpeg/.6.0 # module load SciPy-Stack module load texlive diff --git a/pySDC/projects/GPU/etc/venv_jusuf/requirements.txt b/pySDC/projects/GPU/etc/venv_jusuf/requirements.txt index bd2a63cd8a..d5ddd4873e 100644 --- a/pySDC/projects/GPU/etc/venv_jusuf/requirements.txt +++ b/pySDC/projects/GPU/etc/venv_jusuf/requirements.txt @@ -3,3 +3,4 @@ numpy matplotlib>=3.0 dill>=0.2.6 scipy>=1.14 +tqdm diff --git a/pySDC/projects/GPU/etc/venv_jusuf/setup.sh b/pySDC/projects/GPU/etc/venv_jusuf/setup.sh index 40224178d7..51656618c7 100755 --- a/pySDC/projects/GPU/etc/venv_jusuf/setup.sh +++ b/pySDC/projects/GPU/etc/venv_jusuf/setup.sh @@ -12,7 +12,8 @@ python3 -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}" source "${ABSOLUTE_PATH}"/activate.sh -FFTW_LIBRARY_DIR="/p/software/jusuf/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" python3 -m pip install -e /p/project/ccstma/baumann7/mpi4py-fft + +FFTW_LIBRARY_DIR="/p/software/jusuf/stages/2025/software/FFTW/3.3.10-GCC-13.3.0/lib64/" python3 -m pip install -e /p/project1/ccstma/baumann7/mpi4py-fft python3 -m pip install -e /p/project1/ccstma/baumann7/qmat python3 -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt python3 -m pip install -e /p/project1/ccstma/baumann7/pySDC/ diff --git a/pySDC/projects/GPU/configs/RBC3D_configs.py b/pySDC/projects/RayleighBenard/RBC3D_configs.py similarity index 96% rename from pySDC/projects/GPU/configs/RBC3D_configs.py rename to pySDC/projects/RayleighBenard/RBC3D_configs.py index 064ed60ea9..92697378d1 100644 --- a/pySDC/projects/GPU/configs/RBC3D_configs.py +++ b/pySDC/projects/RayleighBenard/RBC3D_configs.py @@ -212,6 +212,15 @@ def get_description(self, *args, **kwargs): return desc +class RBC3DM2K2(RBC3Dverification): + + def get_description(self, *args, **kwargs): + desc = super().get_description(*args, **kwargs) + desc['level_params']['nsweeps'] = 2 + desc['sweeper_params']['num_nodes'] = 2 + return desc + + class RBC3DM3K4(RBC3Dverification): def get_description(self, *args, **kwargs): @@ -256,9 +265,9 @@ def get_description(self, *args, res=-1, dt=-1, **kwargs): return desc -class RBC3DG4R4Ra1e5(RBC3Dverification): +class RBC3DG4R4SDC22Ra1e5(RBC3DM2K2): Tend = 200 - dt = 6e-2 + dt = 4e-2 res = 32 converged = 50 diff --git a/pySDC/projects/GPU/analysis_scripts/RBC3D_order.py b/pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_order.py similarity index 90% rename from pySDC/projects/GPU/analysis_scripts/RBC3D_order.py rename to pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_order.py index 9bae3c1121..318a599079 100644 --- a/pySDC/projects/GPU/analysis_scripts/RBC3D_order.py +++ b/pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_order.py @@ -2,15 +2,15 @@ import pickle import numpy as np from pySDC.helpers.fieldsIO import FieldsIO -from pySDC.projects.GPU.configs.base_config import get_config +from pySDC.projects.RayleighBenard.RBC3D_configs import get_config from pySDC.implementations.problem_classes.RayleighBenard3D import RayleighBenard3D from mpi4py import MPI import matplotlib.pyplot as plt from pySDC.helpers.plot_helper import figsize_by_journal -from pySDC.projects.GPU.analysis_scripts.RBC3D_plotting_utils import get_plotting_style, savefig +from pySDC.projects.RayleighBenard.analysis_scripts.plotting_utils import get_plotting_style, savefig step_sizes = { - 'RBC3DG4R4Ra1e5': [8e-2, 4e-2, 2e-2, 1e-2, 5e-3], + 'RBC3DG4R4SDC22Ra1e5': [5e-3 * 2**i for i in range(8)], 'RBC3DG4R4SDC23Ra1e5': [5e-3 * 2**i for i in range(8)], 'RBC3DG4R4SDC34Ra1e5': [5e-3 * 2**i for i in range(8)], 'RBC3DG4R4SDC44Ra1e5': [5e-3 * 2**i for i in range(8)], @@ -74,7 +74,7 @@ def plot_error_all_components(args): # pragma: no cover def compare_order(Ra): # pragma: no cover fig, ax = plt.subplots(figsize=figsize_by_journal('Nature_CS', 1, 0.6)) if Ra == 1e5: - names = ['RK', 'Euler', 'SDC23', 'SDC34', 'SDC44'][::-1] + names = ['RK', 'Euler', 'SDC22', 'SDC23', 'SDC34', 'SDC44'][::-1] configs = [f'RBC3DG4R4{me}Ra1e5' for me in names] paths = [f'./data/RBC3DG4R4{me}Ra1e5-res-1-order.pickle' for me in names] @@ -92,7 +92,7 @@ def compare_order(Ra): # pragma: no cover ax.loglog(dt, e, **get_plotting_style(config)) for _dt in dt: - for i in [1, 3, 4]: + for i in [1, 2, 3, 4]: ax.text(_dt, _dt**i, i, fontweight='bold', fontsize=14, ha='center', va='center') ax.loglog(dt, dt**i, ls=':', color='black') @@ -103,7 +103,7 @@ def compare_order(Ra): # pragma: no cover def run(args, dt, Tend): - from pySDC.projects.GPU.run_experiment import run_experiment + from pySDC.projects.RayleighBenard.run_experiment import run_experiment from pySDC.core.errors import ConvergenceError args['mode'] = 'run' @@ -116,7 +116,7 @@ def run(args, dt, Tend): prob = desc['problem_class'](**desc['problem_params']) ic_config_name = type(config).__name__ - for name in ['RK', 'Euler', 'O3', 'O4', 'SDC23', 'SDC34', 'SDC44']: + for name in ['RK', 'Euler', 'O3', 'O4', 'SDC23', 'SDC34', 'SDC44', 'SDC22']: ic_config_name = ic_config_name.replace(name, 'SDC34') ic_config = get_config({**args, 'config': ic_config_name}) @@ -134,7 +134,7 @@ def run(args, dt, Tend): if __name__ == '__main__': - from pySDC.projects.GPU.run_experiment import parse_args + from pySDC.projects.RayleighBenard.run_experiment import parse_args args = parse_args() diff --git a/pySDC/projects/GPU/analysis_scripts/RBC3D_spectrum.py b/pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_spectrum.py similarity index 79% rename from pySDC/projects/GPU/analysis_scripts/RBC3D_spectrum.py rename to pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_spectrum.py index 921bb756c6..9195f79532 100644 --- a/pySDC/projects/GPU/analysis_scripts/RBC3D_spectrum.py +++ b/pySDC/projects/RayleighBenard/analysis_scripts/RBC3D_spectrum.py @@ -1,5 +1,9 @@ -from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import get_pySDC_data -from pySDC.projects.GPU.analysis_scripts.RBC3D_plotting_utils import figsize_by_journal, get_plotting_style, savefig +from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import get_pySDC_data +from pySDC.projects.RayleighBenard.analysis_scripts.plotting_utils import ( + figsize_by_journal, + get_plotting_style, + savefig, +) import matplotlib.pyplot as plt diff --git a/pySDC/projects/GPU/analysis_scripts/plot_Nu.py b/pySDC/projects/RayleighBenard/analysis_scripts/plot_Nu.py similarity index 85% rename from pySDC/projects/GPU/analysis_scripts/plot_Nu.py rename to pySDC/projects/RayleighBenard/analysis_scripts/plot_Nu.py index b63cd8762e..d1db544c8b 100644 --- a/pySDC/projects/GPU/analysis_scripts/plot_Nu.py +++ b/pySDC/projects/RayleighBenard/analysis_scripts/plot_Nu.py @@ -3,7 +3,7 @@ import numpy as np from scipy import integrate from pySDC.helpers.plot_helper import figsize_by_journal, setup_mpl -from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import get_pySDC_data +from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import get_pySDC_data setup_mpl() @@ -39,9 +39,9 @@ def plot_Nu(res, dts, config_name, ref, ax, title): # pragma: no cover if any(error > 1e-2): deviates = min(t_i[error > 1e-2]) ax.axvline(deviates, color=last_line.get_color(), ls=':') - print(f'{title} dt={dt} Nu={Nu_mean:.3f}+={Nu_std:.3f}, deviates more than 1% from t={deviates:.2f}') + print(f'{title} dt={dt:.4f} Nu={Nu_mean:.3f}+={Nu_std:.3f}, deviates more than 1% from t={deviates:.2f}') else: - print(f'{title} dt={dt} Nu={Nu_mean:.3f}+={Nu_std:.3f}') + print(f'{title} dt={dt:.4f} Nu={Nu_mean:.3f}+={Nu_std:.3f}') ax.legend(frameon=True, loc='upper left') @@ -50,12 +50,13 @@ def plot_Nu_over_time_Ra1e5(): # pragma: no cover res = 32 - ref_data = get_pySDC_data(res=res, dt=0.01, config_name='RBC3DG4R4Ra1e5') + ref_data = get_pySDC_data(res=res, dt=0.01, config_name='RBC3DG4R4SDC44Ra1e5') plot_Nu(32, [0.06, 0.04, 0.02], 'RBC3DG4R4SDC34Ra1e5', ref_data, Nu_axs[0], 'SDC34') plot_Nu(32, [0.06, 0.05, 0.02, 0.01], 'RBC3DG4R4SDC23Ra1e5', ref_data, Nu_axs[1], 'SDC23') plot_Nu(32, [0.05, 0.04, 0.02, 0.01, 0.005], 'RBC3DG4R4RKRa1e5', ref_data, Nu_axs[2], 'RK443') plot_Nu(32, [0.02, 0.01, 0.005], 'RBC3DG4R4EulerRa1e5', ref_data, Nu_axs[3], 'RK111') + plot_Nu(32, [0.04, 0.02], 'RBC3DG4R4SDC22Ra1e5', ref_data, Nu_axs[3], 'SDC22') Nu_axs[-1].set_xlabel('$t$') Nu_axs[-1].set_ylabel('$Nu$') diff --git a/pySDC/projects/GPU/analysis_scripts/RBC3D_plotting_utils.py b/pySDC/projects/RayleighBenard/analysis_scripts/plotting_utils.py similarity index 81% rename from pySDC/projects/GPU/analysis_scripts/RBC3D_plotting_utils.py rename to pySDC/projects/RayleighBenard/analysis_scripts/plotting_utils.py index 6403c0524f..6466650c65 100644 --- a/pySDC/projects/GPU/analysis_scripts/RBC3D_plotting_utils.py +++ b/pySDC/projects/RayleighBenard/analysis_scripts/plotting_utils.py @@ -1,5 +1,6 @@ from pySDC.helpers.plot_helper import figsize_by_journal, setup_mpl import warnings +import os setup_mpl() @@ -8,7 +9,12 @@ def get_plotting_style(config): # pragma: no cover args = {'color': None, 'ls': None, 'marker': None, 'markersize': 6, 'label': None} - if config == 'RBC3DG4R4SDC23Ra1e5': + if config == 'RBC3DG4R4SDC22Ra1e5': + args['color'] = 'tab:brown' + args['ls'] = '-' + args['marker'] = '3' + args['label'] = 'SDC22' + elif config == 'RBC3DG4R4SDC23Ra1e5': args['color'] = 'tab:blue' args['ls'] = '-' args['marker'] = 'o' @@ -18,7 +24,7 @@ def get_plotting_style(config): # pragma: no cover args['ls'] = '-' args['marker'] = '<' args['label'] = 'SDC34' - elif config in ['RBC3DG4R4SDC44Ra1e5', 'RBC3DG4R4Ra1e5']: + elif config == 'RBC3DG4R4SDC44Ra1e5': args['color'] = 'tab:green' args['ls'] = '-' args['marker'] = 'x' @@ -40,6 +46,8 @@ def get_plotting_style(config): # pragma: no cover def savefig(fig, name, format='pdf', base_path='./plots', **kwargs): # pragma: no cover + os.makedirs(base_path, exist_ok=True) + path = f'{base_path}/{name}.{format}' fig.savefig(path, bbox_inches='tight', **kwargs) print(f'Saved figure {path!r}') diff --git a/pySDC/projects/GPU/analysis_scripts/process_RBC3D_data.py b/pySDC/projects/RayleighBenard/analysis_scripts/process_RBC3D_data.py similarity index 96% rename from pySDC/projects/GPU/analysis_scripts/process_RBC3D_data.py rename to pySDC/projects/RayleighBenard/analysis_scripts/process_RBC3D_data.py index 25e153207e..80ed5cc33b 100644 --- a/pySDC/projects/GPU/analysis_scripts/process_RBC3D_data.py +++ b/pySDC/projects/RayleighBenard/analysis_scripts/process_RBC3D_data.py @@ -1,5 +1,5 @@ -from pySDC.projects.GPU.configs.base_config import get_config -from pySDC.projects.GPU.run_experiment import parse_args +from pySDC.projects.RayleighBenard.RBC3D_configs import get_config +from pySDC.projects.RayleighBenard.run_experiment import parse_args from pySDC.helpers.fieldsIO import FieldsIO import matplotlib.pyplot as plt from tqdm import tqdm @@ -9,10 +9,11 @@ import os -def process_RBC3D_data(base_path='./data/RBC_time_averaged', plot=True, args=None, config=None): +def process_RBC3D_data(base_path='./data/processed', plot=True, args=None, config=None): # prepare problem instance args = args if args else parse_args() comm = MPI.COMM_WORLD + args['procs'] = [1, 1, comm.size] config = config if config else get_config(args) desc = config.get_description(**args) P = desc['problem_class']( @@ -204,7 +205,7 @@ def process_RBC3D_data(base_path='./data/RBC_time_averaged', plot=True, args=Non return path -def get_pySDC_data(res=-1, dt=-1, config_name='RBC3DG4', base_path='data/RBC_time_averaged'): +def get_pySDC_data(res=-1, dt=-1, config_name='RBC3DG4', base_path='data/processed'): path = f'{base_path}/{config_name}-res{res}-dt{dt:.0e}.pickle' with open(path, 'rb') as file: data = pickle.load(file) diff --git a/pySDC/projects/RayleighBenard/environment.yml b/pySDC/projects/RayleighBenard/environment.yml new file mode 100644 index 0000000000..cc09255ef4 --- /dev/null +++ b/pySDC/projects/RayleighBenard/environment.yml @@ -0,0 +1,18 @@ +--- + +name: pySDC +channels: + - conda-forge +dependencies: + - numpy>=1.15.4 + - scipy>=0.17.1 + - matplotlib>=3.0 + - dill>=0.2.6 + - mpich + - mpi4py-fft>=2.0.2 + - mpi4py>=3.0.0 + - tqdm + - pip + - pip: + - qmat>=0.1.8 + - pytest-isolate-mpi diff --git a/pySDC/projects/RayleighBenard/run_experiment.py b/pySDC/projects/RayleighBenard/run_experiment.py new file mode 100644 index 0000000000..0e38622f8b --- /dev/null +++ b/pySDC/projects/RayleighBenard/run_experiment.py @@ -0,0 +1,87 @@ +def parse_args(): + import argparse + + def cast_to_bool(me): + return False if me in ['False', '0', 0] else True + + def str_to_procs(me): + procs = me.split('/') + assert len(procs) == 3 + return [int(p) for p in procs] + + parser = argparse.ArgumentParser() + parser.add_argument('--useGPU', type=cast_to_bool, help='Toggle for GPUs', default=False) + parser.add_argument( + '--mode', + type=str, + help='Mode for this script', + default=None, + choices=['run', 'plot'], + ) + parser.add_argument('--config', type=str, help='Configuration to load', default=None) + parser.add_argument('--restart_idx', type=int, help='Restart from file by index', default=0) + parser.add_argument('--procs', type=str_to_procs, help='Processes in steps/sweeper/space', default='1/1/1') + parser.add_argument('--res', type=int, help='Space resolution along first axis', default=-1) + parser.add_argument('--dt', type=float, help='(Starting) Step size', default=-1) + parser.add_argument( + '--logger_level', type=int, help='Logger level on the first rank in space and in the sweeper', default=15 + ) + parser.add_argument('-o', type=str, help='output path', default='./') + + return vars(parser.parse_args()) + + +def run_experiment(args, config, **kwargs): + import pickle + import os + + from pySDC.implementations.controller_classes.controller_nonMPI import controller_nonMPI + from pySDC.helpers.stats_helper import filter_stats + + type(config).base_path = args['o'] + os.makedirs(f'{args["o"]}/data', exist_ok=True) + + description = config.get_description( + useGPU=args['useGPU'], MPIsweeper=args['procs'][1] > 1, res=args['res'], dt=args['dt'], **kwargs + ) + controller_params = config.get_controller_params(logger_level=args['logger_level']) + + if args['useGPU']: + from pySDC.implementations.hooks.log_timings import GPUTimings + + controller_params['hook_class'].append(GPUTimings) + + assert ( + config.comms[0].size == 1 + ), 'Have not figured out how to do MPI controller with GPUs yet because I need NCCL for that!' + controller = controller_nonMPI(num_procs=1, controller_params=controller_params, description=description) + prob = controller.MS[0].levels[0].prob + + u0, t0 = config.get_initial_condition(prob, restart_idx=args['restart_idx']) + + config.prepare_caches(prob) + + uend, stats = controller.run(u0=u0, t0=t0, Tend=config.Tend) + + combined_stats = filter_stats(stats, comm=config.comm_world) + + if config.comm_world.rank == config.comm_world.size - 1: + path = f'{config.base_path}/data/{config.get_path()}-stats-whole-run.pickle' + with open(path, 'wb') as file: + pickle.dump(combined_stats, file) + print(f'Stored stats in {path}', flush=True) + + return uend + + +if __name__ == '__main__': + from pySDC.projects.RayleighBenard.RBC3D_configs import get_config + + args = parse_args() + + config = get_config(args) + + if args['mode'] == 'run': + run_experiment(args, config) + else: + raise NotImplementedError diff --git a/pySDC/projects/GPU/tests/test_RBC_3D_analysis.py b/pySDC/projects/RayleighBenard/tests/test_RBC_3D_analysis.py similarity index 82% rename from pySDC/projects/GPU/tests/test_RBC_3D_analysis.py rename to pySDC/projects/RayleighBenard/tests/test_RBC_3D_analysis.py index a005851f8a..acd997dcb6 100644 --- a/pySDC/projects/GPU/tests/test_RBC_3D_analysis.py +++ b/pySDC/projects/RayleighBenard/tests/test_RBC_3D_analysis.py @@ -18,7 +18,7 @@ def get_args(path): def get_config(args): - from pySDC.projects.GPU.configs.base_config import get_config + from pySDC.projects.RayleighBenard.RBC3D_configs import get_config config = get_config(args) config.Tend = 1 @@ -27,7 +27,7 @@ def get_config(args): def generate_simulation_file(path, args=None): - from pySDC.projects.GPU.run_experiment import run_experiment + from pySDC.projects.RayleighBenard.run_experiment import run_experiment args = {**get_args(path), **args} if args is not None else get_args(path) config = get_config(args) @@ -37,7 +37,7 @@ def generate_simulation_file(path, args=None): def generate_processed_file(path): - from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import process_RBC3D_data + from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import process_RBC3D_data args = get_args(path) config = get_config(args) @@ -56,7 +56,7 @@ def tmp_processed_data(tmp_sim_data, tmp_path): def test_ic_interpolation(tmp_sim_data, tmp_path): - from pySDC.projects.GPU.run_experiment import run_experiment + from pySDC.projects.RayleighBenard.run_experiment import run_experiment args = get_args(tmp_path) @@ -80,7 +80,7 @@ def test_processing(tmp_processed_data): def test_get_pySDC_data(tmp_processed_data, tmp_path): - from pySDC.projects.GPU.analysis_scripts.process_RBC3D_data import get_pySDC_data + from pySDC.projects.RayleighBenard.analysis_scripts.process_RBC3D_data import get_pySDC_data args = get_args(tmp_path) data = get_pySDC_data(res=args['res'], dt=args['dt'], config_name=args['config'], base_path=tmp_path) @@ -90,7 +90,7 @@ def test_get_pySDC_data(tmp_processed_data, tmp_path): def test_Nu_interpolation(): - from pySDC.projects.GPU.analysis_scripts.plot_Nu import interpolate_NuV_to_reference_times + from pySDC.projects.RayleighBenard.analysis_scripts.plot_Nu import interpolate_NuV_to_reference_times import numpy as np t = sorted(np.random.rand(128)) @@ -115,8 +115,8 @@ def _get_Nu(_t): def test_error_computation(tmp_sim_data, tmp_path): - from pySDC.projects.GPU.analysis_scripts.RBC3D_order import compute_errors, get_path - from pySDC.projects.GPU.configs.RBC3D_configs import RBC3DG4R4SDC34Ra1e5 + from pySDC.projects.RayleighBenard.analysis_scripts.RBC3D_order import compute_errors, get_path + from pySDC.projects.RayleighBenard.RBC3D_configs import RBC3DG4R4SDC34Ra1e5 import numpy as np import pickle