dask

Author: pancetta

pySDC/playgrounds/parallel/dask_test.exe

@@ -0,0 +1,12 @@
+#!/bin/bash -x
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --output=run.out
+#SBATCH --error=run.err
+#SBATCH --time=00:05:00
+#SBATCH --partition=devel
+
+source /p/home/jusers/speck1/juwels/venv/dask/bin/activate
+
+srun --cpu_bind=sockets python dask_test.py
+touch ready

pySDC/playgrounds/parallel/dask_test.py

@@ -0,0 +1,26 @@
+import cupy
+import dask.array as da
+import time
+
+from dask.distributed import LocalCluster, Client
+
+# generate chunked dask arrays of mamy numpy random arrays
+# rs = da.random.RandomState()
+# x = rs.normal(10, 1, size=(5000, 5000), chunks=(1000, 1000))
+#
+# print(f'{x.nbytes / 1e9} GB of data')
+#
+# t0 = time.time()
+# (x + 1)[::2, ::2].sum().compute(scheduler='single-threaded')
+# print(time.time() - t0)
+#
+# t0 = time.time()
+# (x + 1)[::2, ::2].sum().compute(scheduler='threads')
+# print(time.time() - t0)
+
+if __name__ == '__main__':
+    c = LocalCluster(n_workers=2, processes=True, threads_per_worker=24)
+    print(c)
+
+    c = Client()
+    print(c)