Development Version 0.7 (#5)

* Fixed various issues with proper, out-of-source installation

* Turns out, rather than using a custom action, we can use just the type
if we have a single argument constructor (essentially a type cast)

Author: iangneal

README.md

@@ -17,12 +17,12 @@ To install Lapidary:
 
 ```shell
 # Clone repository
-$ git clone https://github.com/efeslab/lapidary.git
+$ git clone https://github.com/efeslab/lapidary.git ./lapidary_repo
 # Setup virtual environment
 $ python3 -m venv virt_env
 $ source virt_env/bin/activate
 # install
-$ pip3 install ./lapidary
+$ pip3 install ./lapidary_repo
 ```
 
 **Note**: due to some [limitations](#current-limitations), it might be necessary to run 
@@ -94,6 +94,8 @@ an automated fashion (every N seconds or every M instructions) or interactively
 
 ```shell
 $ python3 -m lapidary create --help
+# If ./.lapidary.yaml doesn't exist
+$ python3 -m lapidary -c ./lapidary_repo/.lapidary.yaml create --help
 ```
 
 2. Create checkpoints for an arbitrary binary every second:

lapidary/checkpoint/GDBEngine.py

@@ -238,8 +238,7 @@ def _create_gem5_checkpoint(self, debug_mode):
             import IPython
             IPython.embed()
 
-    @classmethod
-    def _interrupt_in(cls, sec):
+    def _interrupt_in(self, sec):
         '''
             Used to pause GDB with running in timed mode so that checkpoints 
             can be made. A little hackish, but it works.
@@ -250,7 +249,10 @@ def _interrupt_in(cls, sec):
         import gdb
         def control_c(pid, seconds):
             sleep(seconds)
-            os.kill(pid, cls.SIGNAL)
+            try:
+                os.kill(pid, self.SIGNAL)
+            except OSError:
+                self.logger.warning('Could not interrupt inferior process %d as it no longer exists' % pid)
 
         ipid = gdb.selected_inferior().pid
         proc = Process(target=control_c, args=(ipid, sec))

lapidary/config/Gem5FlagConfig.py

@@ -17,6 +17,7 @@ class Gem5FlagConfig:
 
     @classmethod
     def parse_plugins(cls, config):
+        assert isinstance(config, dict)
         if 'gem5_flag_config_plugin' not in config:
             return
 

lapidary/config/LapidaryConfig.py

@@ -9,9 +9,14 @@
 from lapidary.config.Gem5FlagConfig import Gem5FlagConfig
 
 SCHEMA_FILE = Path(__file__).parent / 'schema.yaml'
-SCHEMA = ''
-with SCHEMA_FILE.open('r') as f:
-    SCHEMA = yaml.safe_load(f)
+SCHEMA = None
+
+def get_schema():
+    global SCHEMA
+    if SCHEMA is None:
+        with SCHEMA_FILE.open('r') as f:
+            SCHEMA = yaml.safe_load(f)
+    return SCHEMA
 
 class ConfigException(Exception):
     pass
@@ -22,19 +27,9 @@ def __init__(self, option_strings, dest, **kwargs):
         super().__init__(option_strings, dest, **kwargs)
 
     def __call__(self, parser, namespace, values, option_string=None):
-        print(yaml.dump(SCHEMA))
+        print(yaml.dump(get_schema()))
         exit(0)
 
-class LapidaryConfigHandler(Action):
-    def __init__(self, option_strings, dest, **kwargs):
-        super().__init__(option_strings, dest, **kwargs)
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        filename = getattr(namespace, self.dest)
-        config = self.type(filename=filename)
-        setattr(namespace, self.dest, config)
-        Gem5FlagConfig.parse_plugins(config)
-
 class LapidaryConfig(dict):
 
     def _parse_yaml_data(self, schema, raw_config):
@@ -55,13 +50,12 @@ def _parse_yaml_data(self, schema, raw_config):
                     raw_path = Path(self.filename).parent / element
                     element = raw_path.resolve()
             if isinstance(element, dict):
-                options = SCHEMA[field]['valid_options']
+                options = get_schema()[field]['valid_options']
                 element = self._parse_yaml_data(options, element)
 
             elements[field] = element
 
-        return elements
-        
+        return elements    
 
     def __init__(self, filename=None, rawdata=None):
         assert filename is not None or rawdata is not None
@@ -76,16 +70,19 @@ def __init__(self, filename=None, rawdata=None):
         if not raw_config:
             raise ConfigException('Empty configuration was provided!')
 
-        parsed_config = self._parse_yaml_data(SCHEMA, raw_config)
+        parsed_config = self._parse_yaml_data(get_schema(), raw_config)
         try:
             super().__init__(**parsed_config)
         except:
             super(type(self), self).__init__(**parsed_config)
+        
+        Gem5FlagConfig.parse_plugins(self)
+
 
     @classmethod
     def add_config_arguments(cls, parser):
         parser.add_argument('--config', '-c', default='.lapidary.yaml',
-                            action=LapidaryConfigHandler, type=cls,
+                            type=cls,
                             help=('Load simulation configurations from the '
                                   'specified YAML file.'))
         parser.add_argument('--config-help', action=LapidaryConfigHelp,

lapidary/report/graph/__init__.py

@@ -308,13 +308,16 @@ def add_experiment_args(parser):
     Gem5FlagConfig.add_parser_args(parser)
     SpecBench.add_parser_args(parser)
 
-def do_experiment(args):
+def do_experiment(args, config=None):
 
     global gem5_dir
     global gem5_opt
     global gem5_debug
     global pythonpath
 
+    if config is not None:
+        args.config = config
+
     gem5_dir    = args.config['gem5_path']
     gem5_opt    = gem5_dir / 'build' / 'X86' / 'gem5.opt'
     gem5_debug  = gem5_dir / 'build' / 'X86' / 'gem5.debug'

lapidary/simulate/Experiment.py

@@ -29,6 +29,8 @@ def __init__(self, args, append_log_file=False):
         if not output_dir_parent.exists():
             output_dir_parent.mkdir()
 
+        self.args = args
+
         mode = 'o3'
         if args.in_order:
             mode = 'inorder'
@@ -99,7 +101,6 @@ def __init__(self, args, append_log_file=False):
             output_dir = output_dir_parent / '{}_{}_{}'.format(
                 args.bench, mode, str(chkpt.name))
             arg_list = [
-                './Experiment.py',
                 '--bench', args.bench,
                 '--suite', args.suite,
                 '--warmup-insts', str(args.warmup_insts),
@@ -148,17 +149,16 @@ def __del__(self):
                 json.dump(self.summary, f, indent=4)
 
     @staticmethod
-    def _run_process(experiment_args, log_file):
+    def _run_process(experiment_args, log_file, config):
         '''
             Essentially, we want to just run the experiment stuff again.
             So, we pretend like we're running this from scratch.
 
             This is already a separate process.
         '''
         parser = ArgumentParser()
-        LapidaryConfig.add_config_arguments(parser)
         Experiment.add_experiment_args(parser)
-        args = parser.parse_args(args=experiment_args[1:])   
+        args = parser.parse_args(args=experiment_args)   
 
         prefix = 'lapidary_parallel_simulate'
         with TemporaryFile(mode='w+', prefix=prefix) as out, \
@@ -167,7 +167,7 @@ def _run_process(experiment_args, log_file):
 
             sys.stdout = out
             sys.stderr = err
-            Experiment.do_experiment(args)
+            Experiment.do_experiment(args, config=config)
             out.seek(0)
             err.seek(0)
 
@@ -227,7 +227,7 @@ def do_visual_update(self):
                 remaining_checkpoints = len(self.all_proc_args)
 
                 for chkpt, experiment_args in proc_args.items():
-                    fn_args = (experiment_args, self.log_file)
+                    fn_args = (experiment_args, self.log_file, self.args.config)
                     task = pool.apply_async(ParallelSim._run_process, fn_args)
 
                     tasks[task] = time()
@@ -320,8 +320,11 @@ def add_args(parser):
                             help='Locations of all the checkpoints')
         parser.add_argument('--pool-size', '-p', default=cpu_count(),
                             help='Number of threads to use')
-        parser.add_argument('--log-file', '-l', default='log.txt',
-                            help='Where to log stdout/stderr from experiment runs.')
+        parser.add_argument('--log-file', '-l', 
+            default='parallel_simulation_log_{}.txt'.format(
+                datetime.now().isoformat(sep='_', timespec='seconds')),
+                            help=('Where to log stdout/stderr from experiment runs. '
+                            'Defaults to parallel_simulation_log_<time>.txt'))
         parser.add_argument('--num-checkpoints', '-n', default=None, type=int,
             help='Number of checkpoints to simulate. If None, then all.')
         parser.add_argument('--all-configs', action='store_true',

lapidary/simulate/ParallelSim.py

@@ -137,7 +137,7 @@ def gdb_main():
 #########################################################################
 
 def add_args(parser):
-    LapidaryConfig.add_config_arguments(parser)
+    # LapidaryConfig.add_config_arguments(parser)
     SpecBench.add_parser_args(parser)
     add_arguments(parser)
 
@@ -189,7 +189,7 @@ def main(args):
     arg_list = []
     gdbprocs = []
     if args.cmd:
-        args.cmd[0] = modify_binary_ldd(config, args.cmd[0])
+        args.cmd[0] = modify_binary_ldd(args.config, args.cmd[0])
 
         arg_list = args.cmd
         directory = args.directory if args.directory is not None else '.'

lapidary/simulate/__init__.py

@@ -10,14 +10,16 @@ def main():
 
     setuptools.setup(
         name='lapidary',
-        version='0.0.1',
+        version='0.7.0',
         author='Ian Glen Neal',
-        author_email='ian.gl.neal@gmail.com',
+        author_email='iangneal@umich.com',
         description='A tool for scalable Gem5 Simulation',
         long_description=long_description,
         long_description_content_type='text/markdown',
-        url='',
+        url='https://github.com/efeslab/lapidary',
         packages=setuptools.find_packages(),
+        package_data={'lapidary': ['config/schema.yaml']},
+        setup_requires=['wheel'],
         install_requires=install_requires,
         classifiers=[
             'Programming Language :: Python :: 3',