Setup reproducibility repository

Author: dokempf

README.md

@@ -0,0 +1,19 @@
+This repository documents the software stack and the experimental setup
+of the manuscript: Automatic Code Generation for High-Performance
+Discontinuous Galerkin Methods on Modern Architectures.
+
+It provides the technical details of how the results of the manuscript were
+achieved, but reproduction might require adapting its scripts and configuration
+files to your computing environment.
+
+In general the procedure is:
+* Clone this repository recursively
+* Run ./patch.sh
+* Run ./build_{haswell,skylake}.sh
+* Go into the individual subfolders of build/dune-codegen-paper and apply this procedure to produce a plot:
+  * Run the execution script (like `donkey.sbatch` or `skylake.sh`) 
+    (watch out for requirements on the working directory...)
+  * If there, run `process.sh` to distill the data
+  * Copy the resulting `csv` files into the source and commit them through `git-lfs`
+  * Run `plot.py` in a suitable env (say through `run-in-dune-env`) to generate pdfs.
+

build_haswell.sh

@@ -0,0 +1,20 @@
+ml gcc/6.4.0
+ml benchmark/1.4.0
+ml python/3.6.3
+ml openmpi
+ml cmake
+ml openblas
+ml metis
+ml suite-sparse
+ml superlu
+ml parmetis
+
+SuiteSparse_ROOT=$SUITESPARSE_DIR
+MAKE_FLAGS=-j40 ./dune-common/bin/dunecontrol --module=dune-codegen-paper --builddir=$(pwd)/build --opts=./opts/haswell.opts all
+
+pushd build/dune-codegen-paper
+make -j40 haswell_poisson
+make -j40 haswell_stokes
+make -j40 costmodel_verification_poisson
+make -j40 costmodel_verification_poisson_skeleton
+popd

build_skylake.sh

@@ -0,0 +1,15 @@
+MAKE_FLAGS=-j40 ./dune-common/bin/dunecontrol --opts=skylake.opts --builddir=$(pwd)/build --module=dune-codegen-paper all
+
+# Spin up the autotune build server that runs exclusively on socket 0 and have code generation happen exclusively
+# on socket 1. Communication between the two is realized through the file system, see the server implementation
+hwloc-bind socket:0 ./dune-codegen-paper/autotune-scripts/autotune_build_server.py mpirun --bind-to core -np 20 &
+sleep 0.2
+
+AUTOTUNE_MAKE="hwloc-bind socket:1 make -j20"
+
+pushd build/dune-codegen-paper
+$AUTOTUNE_MAKE skylake_poisson
+$AUTOTUNE_MAKE skylake_stokes
+popd
+
+echo "exit" >> tasks.txt

dune-codegen

@@ -22,10 +22,11 @@ dune_project()
 
 dune_enable_all_packages()
 
-add_subdirectory(src)
-add_subdirectory(dune)
-add_subdirectory(doc)
-add_subdirectory(cmake/modules)
+add_subdirectory(costmodel-verification)
+add_subdirectory(haswell-poisson)
+add_subdirectory(haswell-stokes)
+add_subdirectory(skylake-poisson)
+add_subdirectory(skylake-stokes)
 
 # finalize the dune project, e.g. generating config.h etc.
 finalize_dune_project(GENERATE_CONFIG_H_CMAKE)

dune-codegen-paper/CMakeLists.txt

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+"""
+This is a small server-like application that can run commands that were written
+to a file. This is necessary to separate the code generation process from the
+autotune run environment, using the file system for synchronization.
+"""
+
+import filelock
+import os
+import subprocess
+import sys
+import time
+
+filename = "tasks.txt"
+open(filename, "w").close()
+lock = "{}.lock".format(filename)
+arg_prefix = sys.argv[1:]
+
+keep_running = True
+while keep_running:
+    with filelock.FileLock(lock):
+        with open(filename, "r") as f:
+            lines = f.readlines()
+
+    command = None
+    if lines:
+        command = lines[0].strip("\n")
+
+    if command == "exit":
+        keep_running = False
+        command = None
+
+    if command is not None:
+        subprocess.call(arg_prefix + command.split())
+    
+        with filelock.FileLock(lock):
+            with open(filename, "r") as f:
+                lines = f.readlines()
+
+            with open(filename, "w") as f:
+                for line in lines[1:]:
+                    f.write(line)
+
+    time.sleep(0.1)
+
+os.remove(filename)
+os.remove(lock)

dune-codegen-paper/autotune-scripts/autotune_build_server.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+import filelock
+import sys
+import time
+
+filename = "/home/dkempf/dune-codegen-paper-software/tasks.txt"
+lock = "{}.lock".format(filename)
+command = " ".join(sys.argv[1:])
+
+# Submit the command into the queue
+with filelock.FileLock(lock):
+    with open(filename, "a") as f:
+        f.write("{}\n".format(command))
+
+# Poll the queue for our command still being in
+while True:
+    found = False
+    for line in open(filename, "r"):
+        if command in line:
+            found = True
+
+    if not found:
+        sys.exit(0)
+
+    time.sleep(0.1)

dune-codegen-paper/autotune-scripts/skylake_benchmark_wrapper.py

@@ -0,0 +1,2 @@
+add_subdirectory(poisson_skeleton)
+add_subdirectory(poisson_volume)

dune-codegen-paper/costmodel-verification/CMakeLists.txt

@@ -0,0 +1,5 @@
+dune_add_formcompiler_system_test(UFLFILE poisson_dg_tensor_skeleton.ufl
+                                  BASENAME costmodel_verification_poisson_skeleton
+                                  INIFILE costmodel_verification_poisson_skeleton.mini
+                                  NO_TESTS
+                                  )