Skip to content

Commit 9743f40

Browse files
pancettapancetta
committed
jube runs, vampir traces II
1 parent 07068cf commit 9743f40

File tree

7 files changed

+98
-32
lines changed

7 files changed

+98
-32
lines changed

‎pySDC/projects/Performance/howto_jube.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
module load JUBE
2+
3+
Set up template for the jobscript: run_pySDC_AC.tmpl
4+
Set up xml file for JuBE: jube_run_pySDC_AC.xml
5+
6+
> jube run jube_run_pySDC_AC.xml
7+
> watch jube continue <DIR> --hide-animation --id <ID>
8+
> jube result -a <DIR> --id <ID>

‎pySDC/projects/Performance/jube_run_pySDC_AC.xml‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<jube>
3-
<benchmark name="pySDC AC scaling test" outpath="bench_run_MVA_SPinT">
3+
<benchmark name="pySDC AC scaling test" outpath="bench_run_SPxTS">
44
<comment>Scaling test with pySDC</comment>
55

66
<parameterset name="param_set">
7-
<parameter name="nnodes" type="int">1, 2, 4, 6, 12</parameter>
8-
<!-- <parameter name="ntasks" type="int">1, 2, 4, 6, 12, 24</parameter>-->
9-
<parameter name="ntasks" type="int">24</parameter>
10-
<parameter name="ncpus" type="int">1, 2</parameter>
11-
<parameter name="space_size" mode="python" type="int">$ntasks</parameter>
12-
<!-- <parameter name="compiler_and_mpi" type="str">Intel IntelMPI, Intel ParaStationMPI, GCC MVAPICH2</parameter>-->
13-
<parameter name="compiler_and_mpi" type="str">GCC MVAPICH2</parameter>
14-
<parameter name="flag" type="str">, --cpu_bind=sockets, `./correct_pinning.sh`</parameter>
7+
<parameter name="nnodes" type="int">1</parameter>
8+
<!-- <parameter name="nnodes" type="int">1, 2, 4, 6, 12</parameter>-->
9+
<!-- <parameter name="ntasks" type="int">1, 2, 4, 6, 12</parameter>-->
10+
<parameter name="ntasks" type="int">1</parameter>
11+
<parameter name="ncpus" type="int">1, 2, 4, 12</parameter>
12+
<!-- <parameter name="space_size" mode="python" type="int">$ntasks</parameter>-->
13+
<parameter name="space_size" type="int">1</parameter>
14+
<parameter name="mpi" type="str">psmpi-mt</parameter>
15+
<parameter name="flag" type="str">--cpu_bind=sockets</parameter>
1516
</parameterset>
1617

1718
<!-- Files -->
1819
<fileset name="files">
1920
<copy>run_pySDC_AC.tmpl</copy>
2021
<copy>run_simple_forcing_benchmark.py</copy>
21-
<copy>correct_pinning.sh</copy>
2222
</fileset>
2323

2424
<!-- Substitute -->
@@ -30,7 +30,7 @@
3030
<sub source="#NTASKS#" dest="$ntasks" />
3131
<sub source="#NCPUS#" dest="$ncpus" />
3232
<sub source="#SPACE_SIZE#" dest="$space_size" />
33-
<sub source="#COMPILER_AND_MPI#" dest="$compiler_and_mpi" />
33+
<sub source="#MPI#" dest="$mpi" />
3434
<sub source="#FLAG#" dest="$flag" />
3535
</substituteset>
3636

@@ -60,11 +60,12 @@
6060
<result>
6161
<use>analyse</use> <!-- use existing analyser -->
6262
<table name="result" style="pretty" sort="space_size">
63-
<!-- <column>nnodes</column>-->
63+
<column>nnodes</column>
6464
<column>ntasks</column>
65-
<column>ncpus</column>
66-
<column>flag</column>
65+
<!-- <column>flag</column>-->
6766
<column>space_size</column>
67+
<column>ncpus</column>
68+
<column>mpi</column>
6869
<column>timing_pat</column>
6970
<column>niter_pat</column>
7071
</table>

‎pySDC/projects/Performance/run_pySDC_AC.tmpl‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,45 @@
55
#SBATCH --output=run.out
66
#SBATCH --error=run.err
77
#SBATCH --time=00:05:00
8-
#SBATCH --partition=batch
8+
#SBATCH --partition=devel
99

10-
module load #COMPILER_AND_MPI# SciPy-Stack mpi4py FFTW
10+
export MPI=#MPI#
1111

12-
export LD_PRELOAD=$EBROOTIMKL/mkl/lib/intel64/libmkl_def.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_avx2.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_core.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_intel_lp64.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_intel_thread.so:$EBROOTIMKL/lib/intel64/libiomp5.so
12+
if [ "$MPI" = "intel" ]; then
13+
14+
module load Intel IntelMPI SciPy-Stack mpi4py FFTW
15+
16+
#export I_MPI_ASYNC_PROGRESS=1
17+
#export I_MPI_ASYNC_PROGRESS_THREADS=1
18+
19+
source /p/home/jusers/speck1/jureca/venv/IMPI/bin/activate
20+
21+
fi
22+
23+
if [ "$MPI" = "psmpi" ]; then
24+
25+
module load Intel ParaStationMPI SciPy-Stack mpi4py FFTW
26+
27+
source /p/home/jusers/speck1/jureca/venv/PSMPI/bin/activate
28+
29+
fi
30+
31+
if [ "$MPI" = "psmpi-mt" ]; then
32+
33+
module load Intel ParaStationMPI/5.2.2-1-mt SciPy-Stack mpi4py FFTW
34+
35+
export MPIR_CVAR_ASYNC_PROGRESS=1
36+
export OMP_NUM_THREADS=#NCPUS#
37+
38+
source /p/home/jusers/speck1/jureca/venv/PSMPI-mt/bin/activate
39+
40+
fi
1341

1442
export PYTHONPATH=$PYTHONPATH:/p/project/ccstma/cstma000/pySDC_deploy
1543

1644
export HWT=#NCPUS#
1745
export FLAG=#FLAG#
1846

47+
which python
1948
srun $FLAG python run_simple_forcing_benchmark.py -n #SPACE_SIZE#
20-
#srun $FLAG python run_simple_forcing_benchmark.py -n 1
2149
touch ready

‎pySDC/projects/Performance/run_pySDC_AC_scorep.exe‎

Lines changed: 41 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,64 @@
11
#!/bin/bash -x
22
#SBATCH --nodes=1
3-
#SBATCH --ntasks-per-node=4
3+
#SBATCH --ntasks-per-node=8
44
#SBATCH --cpus-per-task=1
55
#SBATCH --output=run.out
66
#SBATCH --error=run.err
77
#SBATCH --time=00:05:00
88
#SBATCH --partition=devel
99

10-
module unload Score-P
10+
#export MPI=intel
11+
#export MPI=psmpi
12+
export MPI=psmpi-mt
13+
14+
if [ "$MPI" = "intel" ]; then
15+
16+
module load Intel IntelMPI SciPy-Stack mpi4py FFTW PAPI Scalasca
17+
18+
export LD_PRELOAD=/usr/local/software/jureca/Stages/2019a/software/impi/2019.3.199-iccifort-2019.3.199-GCC-8.3.0/intel64/lib/libmpi.so
19+
export PATH=/p/project/ccstma/scorep/6.0-trunk-mprobefix_intel-impi-papi/bin:$PATH
20+
#export I_MPI_ASYNC_PROGRESS=1
21+
#export I_MPI_ASYNC_PROGRESS_THREADS=2
22+
23+
source /p/home/jusers/speck1/jureca/venv/IMPI/bin/activate
24+
25+
fi
26+
27+
if [ "$MPI" = "psmpi" ]; then
28+
29+
module load Intel ParaStationMPI SciPy-Stack mpi4py FFTW PAPI Scalasca
30+
31+
export PATH=/p/project/ccstma/scorep/6.0-trunk-mrobefix_intel-parastation-papi/bin:$PATH
1132

12-
export LD_PRELOAD=$EBROOTIMKL/mkl/lib/intel64/libmkl_def.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_avx2.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_core.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_intel_lp64.so:$EBROOTIMKL/mkl/lib/intel64/libmkl_intel_thread.so:$EBROOTIMKL/lib/intel64/libiomp5.so
33+
source /p/home/jusers/speck1/jureca/venv/PSMPI/bin/activate
1334

14-
export LD_PRELOAD=/usr/local/software/jureca/Stages/2019a/software/impi/2019.3.199-iccifort-2019.3.199-GCC-8.3.0/intel64/lib/libmpi.so
35+
fi
1536

16-
#export PATH=/p/project/ccstma/scorep/6.0-trunk-mrobefix_intel-parastation-papi/bin:$PATH
17-
export PATH=/p/project/ccstma/scorep/6.0-trunk-mprobefix_intel-impi-papi/bin:$PATH
18-
export SCOREP_EXPERIMENT_DIRECTORY=scorep-4x2x1-regions-intel
37+
if [ "$MPI" = "psmpi-mt" ]; then
38+
39+
module load Intel ParaStationMPI/5.2.2-1-mt SciPy-Stack mpi4py FFTW PAPI Scalasca
40+
41+
export PATH=/p/project/ccstma/scorep/6.0-trunk-mrobefix_intel-parastation-papi/bin:$PATH
42+
43+
export MPIR_CVAR_ASYNC_PROGRESS=1
44+
45+
source /p/home/jusers/speck1/jureca/venv/PSMPI-mt/bin/activate
46+
47+
fi
48+
49+
module unload Score-P
50+
51+
export SCOREP_EXPERIMENT_DIRECTORY=scorep-test-$MPI
1952
export SCOREP_PROFILING_MAX_CALLPATH_DEPTH=90
2053
export SCOREP_ENABLE_TRACING=1
2154
#export SCOREP_TOTAL_MEMORY=64M
2255
export SCOREP_METRIC_PAPI=PAPI_TOT_INS
2356

24-
2557
export PYTHONPATH=$PYTHONPATH:/p/project/ccstma/cstma000/pySDC_deploy
26-
#export I_MPI_ASYNC_PROGRESS=1
27-
#export I_MPI_ASYNC_PROGRESS_THREADS=2
28-
#export MPIR_CVAR_ASYNC_PROGRESS=1
2958

3059
#export HWT=2
3160
#export PIN=`./correct_pinning.sh`
3261

33-
srun --cpu_bind=sockets python -m scorep --mpp=mpi run_simple_forcing_benchmark.py -n 1
62+
srun --cpu_bind=sockets python -m scorep --mpp=mpi run_simple_forcing_benchmark.py -n 2
3463
srun scout.mpi --time-correct $SCOREP_EXPERIMENT_DIRECTORY/traces.otf2
3564
touch ready

‎pySDC/projects/Performance/run_simple_forcing_benchmark.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44

55
from pySDC.helpers.stats_helper import filter_stats, sort_stats
66
from pySDC.implementations.collocation_classes.gauss_radau_right import CollGaussRadau_Right
7-
# from pySDC.implementations.controller_classes.controller_MPI import controller_MPI
7+
from pySDC.implementations.controller_classes.controller_MPI import controller_MPI
88
from pySDC.implementations.sweeper_classes.imex_1st_order import imex_1st_order
99
from pySDC.implementations.problem_classes.AllenCahn_MPIFFT import allencahn_imex, allencahn_imex_timeforcing
1010
from pySDC.implementations.transfer_classes.TransferMesh_MPIFFT import fft_to_fft
1111

12-
from pySDC.projects.Performance.controller_MPI_scorep import controller_MPI
12+
# from pySDC.projects.Performance.controller_MPI_scorep import controller_MPI
1313

1414

1515
def run_simulation(name=None, nprocs_space=None):

‎pySDC/projects/Performance/vampir_traces_pySDC_AC_4x1_parastation-mt.png‎

925 KB
Loading

‎pySDC/projects/Performance/vampir_traces_pySDC_AC_4x2_parastation-mt.png‎

1.15 MB
Loading

0 commit comments

Comments
 (0)