Merge branch 'performance'

Author: pancetta

.gitignore

@@ -2,8 +2,6 @@
 /doc/latex
 /doc/html
 
-*.pdf
-
 # Created by https://www.gitignore.io
 
 ### Python ###

pySDC/playgrounds/FEniCS/mass_playground.py

@@ -0,0 +1,44 @@
+import dolfin as df
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+N = 128  # Number of elements
+k = 127  # Wave frequency
+d = 1  # FE order
+
+# Get mesh and function space (CG or DG)
+mesh = df.UnitIntervalMesh(N)
+V = df.FunctionSpace(mesh, "CG", d)
+# V = df.FunctionSpace(mesh, "DG", d)
+
+# Build mass matrix
+u = df.TrialFunction(V)
+v = df.TestFunction(V)
+a_M = u * v * df.dx
+M = df.assemble(a_M)
+
+# Create vector with sine function
+u0 = df.Expression('sin(k*pi*x[0])', pi=np.pi, k=k, degree=d)
+w = df.interpolate(u0, V)
+
+# Apply mass matrix to this vector
+Mw = df.Function(V)
+M.mult(w.vector(), Mw.vector())
+
+# Do FFT to get the frequencies
+fw = np.fft.fft(w.vector()[:])
+fMw = np.fft.fft(Mw.vector()[:])
+# Shift to have zero frequency in the middle of the plot
+fw2 = np.fft.fftshift(fw)
+fMw2 = np.fft.fftshift(fMw)
+
+# Plot
+plt.figure()
+plt.plot(abs(fw2))
+
+plt.figure()
+plt.plot(abs(fMw2))
+
+plt.show()
+

pySDC/playgrounds/parallel/dask_test.exe

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --output=run.out
+#SBATCH --error=run.err
+#SBATCH --time=00:05:00
+#SBATCH --partition=devel
+
+source /p/home/jusers/speck1/juwels/venv/dask/bin/activate
+
+srun --cpu_bind=sockets python dask_test.py
+touch ready

pySDC/playgrounds/parallel/dask_test.py

@@ -0,0 +1,26 @@
+import cupy
+import dask.array as da
+import time
+
+from dask.distributed import LocalCluster, Client
+
+# generate chunked dask arrays of mamy numpy random arrays
+# rs = da.random.RandomState()
+# x = rs.normal(10, 1, size=(5000, 5000), chunks=(1000, 1000))
+#
+# print(f'{x.nbytes / 1e9} GB of data')
+#
+# t0 = time.time()
+# (x + 1)[::2, ::2].sum().compute(scheduler='single-threaded')
+# print(time.time() - t0)
+#
+# t0 = time.time()
+# (x + 1)[::2, ::2].sum().compute(scheduler='threads')
+# print(time.time() - t0)
+
+if __name__ == '__main__':
+    c = LocalCluster(n_workers=2, processes=True, threads_per_worker=24)
+    print(c)
+
+    c = Client()
+    print(c)

pySDC/playgrounds/parallel/isend.py

@@ -0,0 +1,33 @@
+from mpi4py import MPI
+import numpy as np
+import time
+
+
+def sleep(n):
+    tmp = np.random.rand(n)
+
+
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+
+comm.Barrier()
+
+t0 = time.time()
+
+if rank == 0:
+    sbuf = np.empty(40000000)
+    sbuf[0] = 0
+    sbuf[1:4] = np.random.rand(3)
+    req = comm.Isend(sbuf[:], dest=1, tag=99)
+    sleep(100000000)
+    req.wait()
+    print("[%02d] Original data %s" % (rank, sbuf))
+else:
+    rbuf = np.empty(40000000)
+    sleep(10000000)
+    comm.Recv(rbuf[:], source=0, tag=99)
+    print("[%02d] Received data %s" % (rank, rbuf))
+
+t1 = time.time()
+
+print(f'Rank: {rank} -- Time: {t1-t0}')

pySDC/playgrounds/parallel/parallel.batch

@@ -0,0 +1,35 @@
+#!/bin/bash -x
+#SBATCH --nodes=2
+#SBATCH --ntasks-per-node=24
+#SBATCH --cpus-per-task=1
+#SBATCH --output=run.out
+#SBATCH --error=run.err
+#SBATCH --time=00:05:00
+#SBATCH --partition=devel
+
+export HWT=1
+export PIN=`./correct_pinning.sh`
+
+
+#export SCOREP_ENABLE_TRACING=1
+#export PATH=/p/project/ccstma/scorep/6.0-trunk-mrobefix_intel-parastation-papi/bin:$PATH
+#export PATH=/p/project/ccstma/scorep/6.0-trunk-mprobefix_intel-impi-papi/bin:$PATH
+
+#srun python -m scorep --mpp=mpi rma.py
+#srun python -m scorep --mpp=mpi isend.py
+#srun python -m scorep --mpp=mpi thread.py
+
+#srun python rma.py
+#srun python isend.py
+#srun --cpu_bind=sockets python thread.py -n 12
+#srun --cpu_bind=sockets --hint=multithread python thread.py -n 12
+
+echo -e "\n\nDEFAULT PINNING\n---------------------------\n"
+srun --label python thread.py -n 24
+echo -e "\n\nSOCKET PINNING\n---------------------------\n"
+srun --cpu_bind=sockets python thread.py -n 24
+echo -e "\n\nBROEMMEL PINNING\n---------------------------\n"
+srun $PIN --label python thread.py -n 24
+#srun $PIN --label ./show_affinity_jureca.x
+
+touch ready

pySDC/playgrounds/parallel/process.py

@@ -0,0 +1,35 @@
+import mpi4py
+mpi4py.rc.threaded = True
+mpi4py.rc.thread_level = "funneled"
+# mpi4py.rc.profile('vt-hyb', logfile='threads')
+
+from mpi4py import MPI
+from threading import Thread
+
+MPI.COMM_WORLD.Barrier()
+
+# Understanding the Python GIL
+# David Beazley, http://www.dabeaz.com
+# PyCon 2010, Atlanta, Georgia
+# http://www.dabeaz.com/python/UnderstandingGIL.pdf
+
+# Consider this trivial CPU-bound function
+def countdown(n):
+    while n > 0:
+        n -= 1
+
+# Run it once with a lot of work
+COUNT = 10000000 # 10 millon
+tic = MPI.Wtime()
+countdown(COUNT)
+toc = MPI.Wtime()
+print ("sequential: %f seconds" % (toc-tic))
+
+# Now, subdivide the work across two threads
+t1 = Thread(target=countdown, args=(COUNT//2,))
+t2 = Thread(target=countdown, args=(COUNT//2,))
+tic = MPI.Wtime()
+for t in (t1, t2): t.start()
+for t in (t1, t2): t.join()
+toc = MPI.Wtime()
+print ("threaded:   %f seconds" % (toc-tic))

pySDC/playgrounds/parallel/rma.py

@@ -0,0 +1,37 @@
+from mpi4py import MPI
+import numpy as np
+import time
+
+
+def sleep(n):
+    tmp = np.random.rand(n)
+
+
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+
+t0 = time.time()
+
+if rank == 0:
+    sbuf = np.empty(40000000)
+    win = MPI.Win.Create(sbuf, comm=comm)
+    win.Lock(0, MPI.LOCK_EXCLUSIVE)
+    sbuf[0] = 0
+    sbuf[1:4] = np.random.rand(3)
+    win.Unlock(0)
+    sleep(100000000)
+    print("[%02d] Original data %s" % (rank, sbuf))
+else:
+    rbuf = np.empty(40000000)
+    win = MPI.Win.Create(None, comm=comm)
+    sleep(1000000)
+    win.Lock(0, MPI.LOCK_EXCLUSIVE)
+    win.Get(rbuf, 0)
+    win.Unlock(0)
+    print("[%02d] Received data %s" % (rank, rbuf))
+
+t1 = time.time()
+
+win.Free()
+
+print(f'Rank: {rank} -- Time: {t1-t0}')

pySDC/playgrounds/parallel/rma_async.py

@@ -0,0 +1,50 @@
+from mpi4py import MPI
+import numpy as np
+import time
+
+def sleep(n):
+    tmp = np.random.rand(n)
+
+comm = MPI.COMM_WORLD
+rank = comm.Get_rank()
+
+if rank == 0:
+    sbuf = np.empty(4)
+    win = MPI.Win.Create(sbuf, comm=comm)
+else:
+    rbuf = np.empty(4)
+    win = MPI.Win.Create(None, comm=comm)
+    # tmp = np.random.rand(int(10000000/2))
+
+group = win.Get_group()
+
+t0 = time.time()
+
+if rank == 0:
+    sleep(10000000)
+    # tmp = np.random.rand(100000000)
+    for i in range(3):
+        if i > 0:
+            sleep(100000000)
+            win.Wait()
+        sbuf[0] = i
+        sbuf[1:] = np.random.rand(3)
+        print("[%02d] Original data %s" % (rank, sbuf))
+        win.Post(group.Incl([1]))
+    win.Wait()
+else:
+    # tmp = np.random.rand(10000)
+    # tmp = np.random.rand(10000000)
+    # tmp = np.random.rand(1)
+    for i in range(3):
+        win.Start(group.Excl([1]))
+        win.Get(rbuf, 0)
+        win.Complete()
+        sleep(70000000)
+        print("[%02d] Received data %s" % (rank, rbuf))
+
+t1 = time.time()
+group.Free()
+win.Free()
+
+print(f'Rank: {rank} -- Time: {t1-t0}')