util/experiments: Extend with callbacks and parallel SW builds

Author: colluca

experiments/frep/experiments.py

@@ -7,7 +7,7 @@
 
 from copy import deepcopy
 from snitch.target.SimResults import SimRegion
-from snitch.target.experiment_utils import ExperimentManager
+import snitch.target.experiment_utils as eu
 import random
 
 from mako.template import Template
@@ -57,11 +57,10 @@
     'zonl64dobu': str(Path.cwd() / 'hw/zonl64dobu/bin/snitch_cluster.vsim'),
     'zonl48dobu': str(Path.cwd() / 'hw/zonl48dobu/bin/snitch_cluster.vsim'),
 }
-DATA_DIR = Path('data').absolute()
 VERIFY_PY = Path('../../../../sw/blas/gemm/scripts/verify.py').absolute()
 
 
-class FrepExperimentManager(ExperimentManager):
+class FrepExperimentManager(eu.ExperimentManager):
 
     def derive_axes(self, experiment):
         return {
@@ -72,16 +71,7 @@ def derive_axes(self, experiment):
         }
 
     def derive_data_cfg(self, experiment):
-        # Create parent directory for configuration file
-        cfg_path = DATA_DIR / experiment['name'] / 'cfg.json'
-        cfg_path.parent.mkdir(parents=True, exist_ok=True)
-
-        # Fill in configuration template and write configuration file
-        with open('cfg.json.tpl') as f:
-            cfg = Template(f.read()).render(experiment=experiment)
-        with open(cfg_path, 'w') as f:
-            f.write(cfg)
-        return cfg_path
+        return eu.derive_data_cfg_from_template(experiment)
 
     def derive_hw_cfg(self, experiment):
         return Path.cwd() / 'cfg' / f'{experiment["hw"]}.json'

util/experiments/build.py

@@ -60,7 +60,7 @@ def parser():
 
 
 # Build software target with a specific data configuration
-def build(target, build_dir, data_cfg=None, defines=None, hw_cfg=None):
+def build(target=None, build_dir=None, data_cfg=None, defines=None, hw_cfg=None):
     # Define variables for build system
     vars = {
         'DEBUG': 'ON',
@@ -69,16 +69,13 @@ def build(target, build_dir, data_cfg=None, defines=None, hw_cfg=None):
     if data_cfg is not None:
         vars[f'{target}_DATA_CFG'] = data_cfg
     if defines:
-        cflags = ' '.join([f'-D{name}={value}' for name, value in defines.items()])
+        cflags = common.join_cdefines(defines)
         vars[f'{target}_RISCV_CFLAGS'] = cflags
     if hw_cfg is not None:
         vars['CFG_OVERRIDE'] = hw_cfg
 
-    # Build software
-    print(colored('Build app', 'black', attrs=['bold']), colored(target, 'cyan', attrs=['bold']),
-          colored('in', 'black', attrs=['bold']), colored(build_dir, 'cyan', attrs=['bold']))
     env = common.extend_environment(vars)
-    common.make(target, env=env)
+    return common.make(target, env=env, sync=False)
 
 
 # Create test specification for a specific configuration

util/experiments/common.py

@@ -4,13 +4,30 @@
 #
 # Luca Colagrande <colluca@iis.ee.ethz.ch>
 
+import ctypes
 from pathlib import Path
 import os
+import signal
 import subprocess
+import sys
+from termcolor import colored
 
 MK_DIR = Path(__file__).resolve().parent / '../../'
 
 
+# Set the parent-death signal of the current process to `sig`.
+def _set_pdeathsig(sig=signal.SIGTERM):
+    libc = ctypes.CDLL("libc.so.6", use_errno=True)
+    PR_SET_PDEATHSIG = 1
+    if libc.prctl(PR_SET_PDEATHSIG, sig) != 0:
+        e = ctypes.get_errno()
+        raise OSError(e, "prctl(PR_SET_PDEATHSIG) failed")
+
+
+def join_cdefines(defines):
+    return ' '.join([f'-D{key}={value}' for key, value in defines.items()])
+
+
 def extend_environment(vars, env=None):
     if env is None:
         env = os.environ.copy()
@@ -25,9 +42,9 @@ def run(cmd, env=None, dry_run=False, sync=True):
         return None
     else:
         if sync:
-            return subprocess.run(cmd, env=env)
+            return subprocess.run(cmd, env=env, preexec_fn=_set_pdeathsig)
         else:
-            return subprocess.Popen(cmd, env=env)
+            return subprocess.Popen(cmd, env=env, preexec_fn=_set_pdeathsig)
 
 
 def make(target, vars={}, flags=[], dir=MK_DIR, env=None, dry_run=False, sync=True):
@@ -37,3 +54,15 @@ def make(target, vars={}, flags=[], dir=MK_DIR, env=None, dry_run=False, sync=Tr
         cmd.extend(['-C', dir])
     cmd.extend(flags)
     return run(cmd, env=env, dry_run=dry_run, sync=sync)
+
+
+def wait_processes(processes, dry_run=False):
+    if not dry_run:
+        for i, p in enumerate(processes):
+            retcode = p.wait()
+            if retcode != 0:
+                print(
+                    colored(f'Process failed with exit code {retcode}:\n', 'red', attrs=['bold']),
+                    colored(f'{" ".join(p.args)}', 'black')
+                )
+                sys.exit(1)

util/experiments/experiment_utils.py

@@ -39,7 +39,7 @@
 
 class ExperimentManager:
 
-    def __init__(self, experiments=None, actions=None, args=None):
+    def __init__(self, experiments=None, actions=None, args=None, callbacks={}):
         """Initializes the class from the command-line arguments."""
         if args is not None:
             self.args = args
@@ -50,6 +50,9 @@ def __init__(self, experiments=None, actions=None, args=None):
         else:
             self.actions = self.args.actions
 
+        # Save callbacks
+        self.callbacks = callbacks
+
         # Save directory
         self.dir = Path.cwd()
         self.run_dir = self.dir / self.args.run_dir
@@ -146,12 +149,27 @@ def run(self):
 
         # Build software
         if 'sw' in self.actions or 'all' in self.actions:
+            processes = []
             for experiment in experiments:
+                target = experiment['app']
+                build_dir = experiment['elf'].parent
                 defines = self.derive_cdefines(experiment)
                 data_cfg = self.derive_data_cfg(experiment)
                 hw_cfg = self.derive_hw_cfg(experiment)
-                build.build(experiment['app'], experiment['elf'].parent, defines=defines,
-                            data_cfg=data_cfg, hw_cfg=hw_cfg)
+                if 'sw' in self.callbacks:
+                    func = self.callbacks['sw']
+                else:
+                    func = build.build
+                print(colored('Build app', 'black', attrs=['bold']),
+                      colored(target, 'cyan', attrs=['bold']),
+                      colored('in', 'black', attrs=['bold']),
+                      colored(build_dir, 'cyan', attrs=['bold']))
+                process = func(
+                    target=target, build_dir=build_dir, defines=defines,
+                    data_cfg=data_cfg, hw_cfg=hw_cfg, dry_run=dry_run
+                )
+                processes.append(process)
+            common.wait_processes(processes, dry_run=dry_run)
 
         # Run experiments
         if 'run' in self.actions or 'all' in self.actions:
@@ -195,11 +213,7 @@ def run(self):
                 process = common.make('perf', vars, flags=flags, sync=False)
                 processes.append(process)
 
-            # Wait for all processes to complete
-            for i, process in enumerate(processes):
-                return_code = process.wait()
-                if return_code != 0:
-                    raise Exception(f'Failed to generate performance dump for experiment {i}')
+            common.wait_processes(processes)
 
         # Build visual traces
         if 'visual-trace' in self.actions or 'all' in self.actions:
@@ -361,3 +375,46 @@ def get_area_results(self):
         df = pd.concat(columns, axis=1)
 
         return df
+
+
+def derive_axes_from_keys(experiment, keys):
+    """Designate some keys in the experiment as the experiment axes."""
+    return {key: experiment[key] for key in keys}
+
+
+def derive_data_cfg_from_template(experiment, template_path="cfg.json.tpl",
+                                  root_cfg_dir=Path('data').absolute()):
+    """Derive the data configuration file from a template.
+
+    Fills the template file using the experiment dictionary.
+    Writes it to a file under the root data configuration directory, creating
+    a directory hierarchy defined by the experiment name.
+    The output filename is derived from the template's, stripped of the '.tpl'
+    suffix, when this suffix exists.
+
+    Args:
+        experiment: Experiment dictionary.
+        template_path: Path to the template file.
+        root_cfg_dir: Root data configuration directory.
+    """
+    # Read the template file
+    with open(template_path, 'r') as f:
+        template = mako.template.Template(f.read())
+
+    # Fill in (render) the template with the experiment parameters
+    cfg = template.render(experiment=experiment)
+
+    # Derive output filename from template basename by stripping a trailing '.tpl'
+    cfg_name = template_path.stem if template_path.suffix == '.tpl' else template_path.name
+
+    # Compose destination path from the root data configuration directory,
+    # following a directory hierarchy defined by the experiment name
+    cfg_path = root_cfg_dir / experiment["name"] / cfg_name
+    cfg_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write the rendered template to file
+    with open(cfg_path, 'w') as f:
+        f.write(cfg)
+
+    # Return the path to the rendered configuration file
+    return cfg_path