scaling tests and visualization

pancetta · pancetta · commit 285c9a6a7d5b · 2019-08-15T11:38:28.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -2,8 +2,6 @@
 /doc/latex
 /doc/html
 
-*.pdf
-
 # Created by https://www.gitignore.io
 
 ### Python ###
diff --git a/pySDC/playgrounds/FEniCS/mass_playground.py b/pySDC/playgrounds/FEniCS/mass_playground.py
@@ -0,0 +1,44 @@
+import dolfin as df
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+N = 128  # Number of elements
+k = 127  # Wave frequency
+d = 1  # FE order
+
+# Get mesh and function space (CG or DG)
+mesh = df.UnitIntervalMesh(N)
+V = df.FunctionSpace(mesh, "CG", d)
+# V = df.FunctionSpace(mesh, "DG", d)
+
+# Build mass matrix
+u = df.TrialFunction(V)
+v = df.TestFunction(V)
+a_M = u * v * df.dx
+M = df.assemble(a_M)
+
+# Create vector with sine function
+u0 = df.Expression('sin(k*pi*x[0])', pi=np.pi, k=k, degree=d)
+w = df.interpolate(u0, V)
+
+# Apply mass matrix to this vector
+Mw = df.Function(V)
+M.mult(w.vector(), Mw.vector())
+
+# Do FFT to get the frequencies
+fw = np.fft.fft(w.vector()[:])
+fMw = np.fft.fft(Mw.vector()[:])
+# Shift to have zero frequency in the middle of the plot
+fw2 = np.fft.fftshift(fw)
+fMw2 = np.fft.fftshift(fMw)
+
+# Plot
+plt.figure()
+plt.plot(abs(fw2))
+
+plt.figure()
+plt.plot(abs(fMw2))
+
+plt.show()
+
diff --git a/pySDC/projects/AllenCahn_Bayreuth/AllenCahn_dump.py b/pySDC/projects/AllenCahn_Bayreuth/AllenCahn_dump.py
@@ -120,7 +120,7 @@ def post_step(self, step, level_number):
             nbytes_global = [nbytes_local]
         local_offset = sum(nbytes_global[:self.rank])
 
-        #  dump initial data
+        #  dump data
         fname = f"./data/{L.prob.params.name}_{self.time_step + step.status.slot:08d}"
         fh = MPI.File.Open(self.comm, fname + ".dat", self.amode)
         fh.Write_at_all(local_offset, tmp)
diff --git a/pySDC/projects/Performance/data/AC-bench-noforce_00000000.dat b/pySDC/projects/Performance/data/AC-bench-noforce_00000000.dat
diff --git a/pySDC/projects/Performance/data/AC-bench-noforce_00000000.json b/pySDC/projects/Performance/data/AC-bench-noforce_00000000.json
@@ -0,0 +1 @@
+{"type": "dataset", "datatype": "float64", "endian": "=", "time": 0.0, "space_comm_size": 24, "time_comm_size": 1, "shape": [576, 576], "elementsize": 8}
diff --git a/pySDC/projects/Performance/data/AC-bench-noforce_00000024.dat b/pySDC/projects/Performance/data/AC-bench-noforce_00000024.dat
diff --git a/pySDC/projects/Performance/data/AC-bench-noforce_00000024.json b/pySDC/projects/Performance/data/AC-bench-noforce_00000024.json
@@ -0,0 +1 @@
+{"type": "dataset", "datatype": "float64", "endian": "=", "time": 0.024000000000000014, "space_comm_size": 24, "time_comm_size": 1, "shape": [576, 576], "elementsize": 8}
diff --git a/pySDC/projects/Performance/data/bench_run_SPxTP/000002/result/result.dat b/pySDC/projects/Performance/data/bench_run_SPxTP/000002/result/result.dat
@@ -0,0 +1,8 @@
+result:
+nnodes | ntasks | space_size | ncpus | timing_pat | niter_pat
+-------+--------+------------+-------+------------+----------
+     1 |     24 |         24 |     1 |     3.5312 |       4.0
+     2 |     24 |         24 |     1 |     2.4658 |       5.0
+     4 |     24 |         24 |     1 |     1.5581 |       5.5
+     6 |     24 |         24 |     1 |     1.1749 |      5.75
+    12 |     24 |         24 |     1 |     0.9213 |       7.0
diff --git a/pySDC/projects/Performance/data/bench_run_SPxTS/000004/result/result.dat b/pySDC/projects/Performance/data/bench_run_SPxTS/000004/result/result.dat
@@ -0,0 +1,10 @@
+result:
+nnodes | ntasks | space_size | ncpus | timing_pat | niter_pat
+-------+--------+------------+-------+------------+----------
+     1 |      1 |          1 |     1 |    31.6875 |       4.0
+     1 |      2 |          2 |     1 |    18.3877 |       4.0
+     1 |      4 |          4 |     1 |    10.1326 |       4.0
+     1 |      6 |          6 |     1 |     7.1452 |       4.0
+     1 |     12 |         12 |     1 |     4.5086 |       4.0
+     1 |     24 |         24 |     1 |       3.52 |       4.0
+     2 |     24 |         48 |     1 |     3.4735 |       4.0
diff --git a/pySDC/projects/Performance/jube_run_pySDC_AC_space.xml b/pySDC/projects/Performance/jube_run_pySDC_AC_space.xml
@@ -4,14 +4,12 @@
     <comment>Scaling test with pySDC</comment>
 
     <parameterset name="param_set">
-      <parameter name="nnodes" type="int">1</parameter>
-<!--      <parameter name="nnodes" type="int">1, 2, 4, 6, 12</parameter>-->
-<!--      <parameter name="ntasks" type="int">1, 2, 4, 6, 12</parameter>-->
-      <parameter name="ntasks" type="int">1</parameter>
-      <parameter name="ncpus" type="int">1, 2, 4, 12</parameter>
-<!--      <parameter name="space_size" mode="python" type="int">$ntasks</parameter>-->
-      <parameter name="space_size" type="int">1</parameter>
-      <parameter name="mpi" type="str">psmpi-mt</parameter>
+      <parameter name="i">0, 1, 2, 3, 4, 5, 6</parameter>
+      <parameter name="nnodes" mode="python" type="int">[1, 1, 1, 1,  1,  1,  2][$i]</parameter>
+      <parameter name="ntasks" mode="python" type="int">[1, 2, 4, 6, 12, 24, 24][$i]</parameter>
+      <parameter name="ncpus" type="int">1</parameter>
+      <parameter name="space_size" mode="python" type="int">$ntasks * $nnodes</parameter>
+      <parameter name="mpi" type="str">intel</parameter>
       <parameter name="flag" type="str">--cpu_bind=sockets</parameter>
     </parameterset>
 
@@ -65,7 +63,7 @@
 <!--        <column>flag</column>-->
         <column>space_size</column>
         <column>ncpus</column>
-        <column>mpi</column>
+<!--        <column>mpi</column>-->
         <column>timing_pat</column>
         <column>niter_pat</column>
       </table>
diff --git a/pySDC/projects/Performance/jube_run_pySDC_AC_spacetime.xml b/pySDC/projects/Performance/jube_run_pySDC_AC_spacetime.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<jube>
+  <benchmark name="pySDC AC scaling test" outpath="bench_run_SPxTP">
+    <comment>Scaling test with pySDC</comment>
+
+    <parameterset name="param_set">
+      <parameter name="i">0, 1, 2, 3, 4</parameter>
+      <parameter name="nnodes" mode="python" type="int">[ 1,  1,  2,  4,  6][$i]</parameter>
+      <parameter name="ntasks" mode="python" type="int">[12, 24, 24, 24, 24][$i]</parameter>
+      <parameter name="ncpus" type="int">1</parameter>
+      <parameter name="space_size" mode="python" type="int">$ntasks</parameter>
+      <parameter name="mpi" type="str">intel</parameter>
+      <parameter name="flag" type="str">--cpu_bind=sockets</parameter>
+    </parameterset>
+
+    <!-- Files -->
+    <fileset name="files">
+      <copy>run_pySDC_AC.tmpl</copy>
+      <copy>run_simple_forcing_benchmark.py</copy>
+    </fileset>
+
+    <!-- Substitute -->
+    <substituteset name="substitute">
+      <!-- Substitute files -->
+      <iofile in="run_pySDC_AC.tmpl" out="run_pySDC_AC.exe" />
+      <!-- Substitute commands -->
+      <sub source="#NNODES#" dest="$nnodes" />
+      <sub source="#NTASKS#" dest="$ntasks" />
+      <sub source="#NCPUS#" dest="$ncpus" />
+      <sub source="#SPACE_SIZE#" dest="$space_size" />
+      <sub source="#MPI#" dest="$mpi" />
+      <sub source="#FLAG#" dest="$flag" />
+    </substituteset>
+
+    <!-- Regex pattern -->
+    <patternset name="pattern">
+      <pattern name="timing_pat" type="float">Time to solution: $jube_pat_fp sec.</pattern>
+      <pattern name="niter_pat" type="float">Mean number of iterations: $jube_pat_fp</pattern>
+    </patternset>
+    
+    <!-- Operation -->
+    <step name="sub_step">
+      <use>param_set</use> <!-- use existing parameterset -->
+      <use>files</use>        <!-- use existing fileset -->
+      <use>substitute</use>   <!-- use existing substituteset -->
+      <do done_file="ready">sbatch -A cstma run_pySDC_AC.exe</do>   <!-- shell command -->
+    </step>
+
+    <!-- Analyse -->
+    <analyser name="analyse">
+      <use>pattern</use> <!-- use existing patternset -->
+      <analyse step="sub_step">
+        <file>run.out</file> <!-- file which should be scanned -->
+      </analyse>
+    </analyser>
+
+    <!-- Create result table -->
+    <result>
+      <use>analyse</use> <!-- use existing analyser -->
+      <table name="result" style="pretty" sort="space_size">
+        <column>nnodes</column>
+        <column>ntasks</column>
+<!--        <column>flag</column>-->
+        <column>space_size</column>
+        <column>ncpus</column>
+<!--        <column>mpi</column>-->
+        <column>timing_pat</column>
+        <column>niter_pat</column>
+      </table>
+    </result>
+
+  </benchmark>
+</jube>
diff --git a/pySDC/projects/Performance/run_pySDC_AC.exe b/pySDC/projects/Performance/run_pySDC_AC.exe
@@ -0,0 +1,18 @@
+#!/bin/bash -x
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=24
+#SBATCH --cpus-per-task=1
+#SBATCH --output=run.out
+#SBATCH --error=run.err
+#SBATCH --time=00:30:00
+#SBATCH --partition=batch
+
+module load Intel IntelMPI SciPy-Stack mpi4py FFTW
+
+source /p/home/jusers/speck1/jureca/venv/IMPI/bin/activate
+
+export PYTHONPATH=$PYTHONPATH:/p/project/ccstma/cstma000/pySDC_deploy
+
+which python
+srun --cpu_bind=sockets python run_simple_forcing_benchmark.py -n 24
+touch ready
diff --git a/pySDC/projects/Performance/run_pySDC_AC.tmpl b/pySDC/projects/Performance/run_pySDC_AC.tmpl
@@ -5,7 +5,7 @@
 #SBATCH --output=run.out
 #SBATCH --error=run.err
 #SBATCH --time=00:05:00
-#SBATCH --partition=devel
+#SBATCH --partition=batch
 
 export MPI=#MPI#
 
diff --git a/pySDC/projects/Performance/run_pySDC_AC_scorep.exe b/pySDC/projects/Performance/run_pySDC_AC_scorep.exe
@@ -48,8 +48,7 @@ fi
 
 module unload Score-P
 
-#export SCOREP_EXPERIMENT_DIRECTORY=scorep-test-$MPI
-export SCOREP_EXPERIMENT_DIRECTORY=scorep-4x2x1-regions-$MPI
+export SCOREP_EXPERIMENT_DIRECTORY=data/scorep-4x2x1-regions-$MPI
 export SCOREP_PROFILING_MAX_CALLPATH_DEPTH=90
 export SCOREP_ENABLE_TRACING=1
 #export SCOREP_TOTAL_MEMORY=64M
diff --git a/pySDC/projects/Performance/run_simple_forcing_benchmark.py b/pySDC/projects/Performance/run_simple_forcing_benchmark.py
@@ -4,12 +4,13 @@
 
 from pySDC.helpers.stats_helper import filter_stats, sort_stats
 from pySDC.implementations.collocation_classes.gauss_radau_right import CollGaussRadau_Right
-# from pySDC.implementations.controller_classes.controller_MPI import controller_MPI
+from pySDC.implementations.controller_classes.controller_MPI import controller_MPI
 from pySDC.implementations.sweeper_classes.imex_1st_order import imex_1st_order
 from pySDC.implementations.problem_classes.AllenCahn_MPIFFT import allencahn_imex, allencahn_imex_timeforcing
 from pySDC.implementations.transfer_classes.TransferMesh_MPIFFT import fft_to_fft
+from pySDC.projects.AllenCahn_Bayreuth.AllenCahn_dump import dump
 
-from pySDC.projects.Performance.controller_MPI_scorep import controller_MPI
+# from pySDC.projects.Performance.controller_MPI_scorep import controller_MPI
 
 
 def run_simulation(name=None, nprocs_space=None):
@@ -82,6 +83,7 @@ def run_simulation(name=None, nprocs_space=None):
     controller_params = dict()
     controller_params['logger_level'] = 30 if space_rank == 0 else 99  # set level depending on rank
     controller_params['predict_type'] = 'fine_only'
+    controller_params['hook_class'] = dump  # activate to get data output at each step
 
     # fill description dictionary for easy step instantiation
     description = dict()
@@ -101,7 +103,7 @@ def run_simulation(name=None, nprocs_space=None):
 
     # set time parameters
     t0 = 0.0
-    Tend = 12 * 0.001
+    Tend = 240 * 0.001
 
     if space_rank == 0 and time_rank == 0:
         out = f'---------> Running {name} with {time_size} process(es) in time and {space_size} process(es) in space...'
diff --git a/pySDC/projects/Performance/visualize.py b/pySDC/projects/Performance/visualize.py
@@ -0,0 +1,88 @@
+import glob
+import json
+import numpy as np
+import pySDC.helpers.plot_helper as plt_helper
+
+
+def joint_plots(list_of_result_paths):
+
+    # get result data from JUBE tables
+    results = []
+    for result_path in list_of_result_paths:
+        results.append(np.genfromtxt(result_path, names=True, skip_header=1, delimiter='|', dtype=float, comments='--'))
+
+    # fill arrays with data
+    ncores = np.concatenate(results)['ntasks'] * np.concatenate(results)['nnodes']
+    timings_space = results[0]['timing_pat']
+    timings_spacetime = results[1]['timing_pat']
+    ideal = [timings_space[0] / (c / ncores[0]) for c in np.unique(ncores)]
+
+    # setup and fill plots
+    plt_helper.setup_mpl()
+    plt_helper.newfig(textwidth=238.96, scale=1.0)
+
+    plt_helper.plt.loglog(np.unique(ncores), ideal, 'k--', label='ideal')
+    plt_helper.plt.loglog(ncores[0:len(results[0])], timings_space, lw=1, ls='-', color='b', marker='o',
+                          markersize=4, markeredgecolor='k', label='parallel-in-space')
+    plt_helper.plt.loglog(ncores[-len(results[1]):], timings_spacetime, lw=1, ls='-', color='r', marker='d',
+                          markersize=4, markeredgecolor='k', label='parallel-in-space-time')
+
+    plt_helper.plt.grid()
+    plt_helper.plt.legend(loc=3, ncol=1)
+    plt_helper.plt.xlabel('Number of cores')
+    plt_helper.plt.ylabel('Time [s]')
+
+    # save plot, beautify
+    fname = 'data/scaling'
+    plt_helper.savefig(fname)
+
+
+def plot_data(name=''):
+    """
+    Visualization using numpy arrays (written via MPI I/O) and json description
+
+    Produces one png file per time-step, combine as movie via e.g.
+      > ffmpeg -i data/name_%08d.png name.mp4
+
+    Args:
+        name (str): name of the simulation (expects data to be in data path)
+    """
+
+    # get data and json files
+    json_files = sorted(glob.glob(f'./data/{name}_*.json'))
+    data_files = sorted(glob.glob(f'./data/{name}_*.dat'))
+
+    # setup plotting
+    plt_helper.setup_mpl()
+
+    for json_file, data_file in zip(json_files, data_files):
+        with open(json_file, 'r') as fp:
+            obj = json.load(fp)
+
+        index = json_file.split('_')[1].split('.')[0]
+        print(f'Working on step {index}...')
+
+        # get data and format
+        array = np.fromfile(data_file, dtype=obj['datatype'])
+        array = array.reshape(obj['shape'], order='C')
+
+        # plot
+        plt_helper.newfig(textwidth=238.96, scale=1.0)
+        plt_helper.plt.imshow(array, vmin=0, vmax=1, extent=[-2, 2, -2, 2], origin='lower')
+
+        plt_helper.plt.yticks(range(-2, 3))
+        cbar = plt_helper.plt.colorbar()
+        cbar.set_label('concentration')
+        plt_helper.plt.title(f"Time: {obj['time']:6.4f}")
+
+        # save plot, beautify
+        fname = f'data/{name}_{index}'
+        plt_helper.savefig(fname, save_pgf=False, save_png=False)
+
+
+if __name__ == '__main__':
+    list_of_result_paths = ['data/bench_run_SPxTS/000004/result/result.dat',
+                            'data/bench_run_SPxTP/000002/result/result.dat']
+
+    # joint_plots(list_of_result_paths)
+    plot_data(name='AC-bench-noforce')

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"type": "dataset", "datatype": "float64", "endian": "=", "time": 0.0, "space_comm_size": 24, "time_comm_size": 1, "shape": [576, 576], "elementsize": 8}`