Adds an analysis of the sub-tracked code in addition to a comparison of the analysis between tracking sessions

Author: erikhuck

src/gpu_tracker/__init__.py

@@ -10,5 +10,4 @@
     __version__ = _gv(_path.join(_path.dirname(__file__), _path.pardir))
 
 from .tracker import Tracker
-from .sub_tracker import SubTracker
-from .sub_tracker import sub_track
+from .sub_tracker import SubTracker, sub_track, SubTrackingAnalyzer, TrackingComparison

src/gpu_tracker/__main__.py

@@ -4,14 +4,18 @@
 Usage:
     gpu-tracker -h | --help
     gpu-tracker -v | --version
-    gpu-tracker --execute=<command> [--output=<output>] [--format=<format>] [--st=<sleep-time>] [--ru=<ram-unit>] [--gru=<gpu-ram-unit>] [--tu=<time-unit>] [--nec=<num-cores>] [--guuids=<gpu-uuids>] [--disable-logs] [--gb=<gpu-brand>] [--tf=<tracking-file>]
+    gpu-tracker --execute=<command> [--output=<output>] [--format=<format>] [--tconfig=<config-file>] [--st=<sleep-time>] [--ru=<ram-unit>] [--gru=<gpu-ram-unit>] [--tu=<time-unit>] [--nec=<num-cores>] [--guuids=<gpu-uuids>] [--disable-logs] [--gb=<gpu-brand>] [--tf=<tracking-file>] [--overwrite]
+    gpu-tracker sub-track combine --stf=<sub-track-file> -p <file-path>...
+    gpu-tracker sub-track analyze --tf=<tracking-file> --stf=<sub-track-file> [--output=<output>] [--format=<format>]
+    gpu-tracker sub-track compare [--output=<output>] [--format=<format>] [--cconfig=<config-file>] [-m <name>=<file-path>...] [--stat=<statistic>]
 
 Options:
     -h --help               Show this help message and exit.
     -v --version            Show package version and exit.
     -e --execute=<command>  The command to run along with its arguments all within quotes e.g. "ls -l -a".
-    -o --output=<output>    File path to store the computational-resource-usage measurements. If not set, prints measurements to the screen.
-    -f --format=<format>    File format of the output. Either 'json' or 'text'. Defaults to 'text'.
+    -o --output=<output>    File path to store the computational-resource-usage measurements in the case of tracking or the analysis report in the case of sub-tracking. If not set, prints to the screen.
+    -f --format=<format>    File format of the output. Either 'json', 'text', or 'pickle'. Defaults to 'text'.
+    --tconfig=<config-file> JSON config file containing the key word arguments to the ``Tracker`` class (see API) to be optionally used instead of the corresponding commandline options. If any commandline options are set, they will override the corresponding arguments provided by the config file.
     --st=<sleep-time>       The number of seconds to sleep in between usage-collection iterations.
     --ru=<ram-unit>         One of 'bytes', 'kilobytes', 'megabytes', 'gigabytes', or 'terabytes'.
     --gru=<gpu-ram-unit>    One of 'bytes', 'kilobytes', 'megabytes', 'gigabytes', or 'terabytes'.
@@ -21,65 +25,134 @@
     --disable-logs          If set, warnings are suppressed during tracking. Otherwise, the Tracker logs warnings as usual.
     --gb=<gpu-brand>        The brand of GPU to profile. Valid values are nvidia and amd. Defaults to the brand of GPU detected in the system, checking NVIDIA first.
     --tf=<tracking-file>    If specified, stores the individual resource usage measurements at each iteration. Valid file formats are CSV (.csv) and SQLite (.sqlite) where the SQLite file format stores the data in a table called "data" and allows for more efficient querying.
+    --overwrite             Whether to overwrite the tracking file if it already existed before the beginning of this tracking session. Do not set if the data in the existing tracking file is still needed.
+    sub-track               Perform sub-tracking related commands.
+    combine                 Combines multiple sub-tracking files into one. This is usually a result of sub-tracking a code block that is called in multiple simultaneous processes.
+    --stf=<sub-track-file>  The path to the sub-tracking file used to specify the timestamps of specific code-blocks. If not generated by the gpu-tracker API, must be either a CSV or SQLite file (where the SQLite file contains a table called "data") where the headers are precisely process_id, code_block_name, position, and timestamp. The process_id is the ID of the process where the code block is called. code_block_name is the name of the code block. position is whether it is the start or the stopping point of the code block where 0 represents start and 1 represents stop. And timestamp is the timestamp where the code block starts or where it stops.
+    -p <file-path>          Paths to the sub-tracking files to combine. Must all be the same file format and the same file format as the resulting sub-tracking file (either .csv or .sqlite). If only one path is provided, it is interpreted as a path to a directory and all the files in this directory are combined.
+    analyze                 Generate the sub-tracking analysis report using the tracking file and sub-tracking file for resource usage of specific code blocks.
+    compare                 Compares multiple tracking sessions to determine differences in computational resource usage by loading sub-tracking results given their file paths. Sub-tracking results files must be in pickle format e.g. running the ``sub-track analyze`` command and specifying a file path for ``--output`` and 'pickle' for the ``--format`` option. If code block results are not included in the sub-tracking files (i.e. no code blocks were sub-tracked), then only overall results are compared.
+    --cconfig=<config-file> JSON config file containing the ``file_path_map`` argument for the ``TrackerComparison`` class and ``statistic`` argument for its ``compare`` method (see API) that can be used instead of the corresponding ``-m <name>=<path>`` and ``--stat=<statistic>`` commandline options respectively. If additional ``-m <name>=<path>`` options are added on the commandline in addition to a config file, they will be added to the ``file_path_map`` in the config file. If a ``--stat`` option is provided on the commandline, it will override the ``statistic`` in the config file.
+    -m <name>=<file-path>   Mapping of tracking session names to the path of the file containing the sub-tracking results of said tracking session. Must be in pickle format.
+    --stat=<statistic>      The summary statistic of the measurements to compare. One of 'min', 'max', 'mean', or 'std'. Defaults to 'mean'.
 """
 import docopt as doc
 import subprocess as subp
 import json
 import logging as log
 import sys
-from . import Tracker
-from . import __version__
+import os
+import pickle as pkl
+from . import Tracker, SubTrackingAnalyzer, TrackingComparison, __version__
 
 
 def main():
     args = doc.docopt(__doc__, version=__version__)
-    command = args['--execute'].split()
     output = args['--output']
     output_format = args['--format'] if args['--format'] is not None else 'text'
-    option_map = {
-        '--st': 'sleep_time',
-        '--ru': 'ram_unit',
-        '--gru': 'gpu_ram_unit',
-        '--tu': 'time_unit',
-        '--nec': 'n_expected_cores',
-        '--guuids': 'gpu_uuids',
-        '--disable-logs': 'disable_logs',
-        '--gb': 'gpu_brand',
-        '--tf': 'tracking_file'
-    }
-    kwargs = {
-        option_map[option]: value for option, value in args.items() if value is not None and option not in {
-            '--execute', '--output', '--format', '--help', '--version'}}
-    if 'sleep_time' in kwargs.keys():
-        kwargs['sleep_time'] = float(kwargs['sleep_time'])
-    if 'n_expected_cores' in kwargs.keys():
-        kwargs['n_expected_cores'] = int(kwargs['n_expected_cores'])
-    if 'gpu_uuids' in kwargs.keys():
-        kwargs['gpu_uuids'] = set(kwargs['gpu_uuids'].split(','))
-    if len(command) == 0:
-        log.error('Empty command provided.')
-        sys.exit(1)
-    try:
-        process = subp.Popen(command)
-    except FileNotFoundError:
-        log.error(f'Command not found: "{command[0]}"')
-        sys.exit(1)
-    except Exception as e:
-        log.error(f'The following error occurred when starting the command "{command[0]}":')
-        print(e)
-        sys.exit(1)
-    with Tracker(process_id=process.pid, **kwargs) as tracker:
-        process.wait()
-    print(f'Resource tracking complete. Process completed with status code: {process.returncode}')
+    if args['sub-track']:
+        if args['analyze']:
+            tracking_file = args['--tf']
+            sub_tracking_file = args['--stf']
+            results = SubTrackingAnalyzer(tracking_file, sub_tracking_file).sub_tracking_results()
+            _process_output(output_format, results, output)
+        elif args['combine']:
+            directory = None
+            files = None
+            files = args['-p']
+            if len(files) == 1:
+                [directory] = files
+                files = [os.path.join(directory, file) for file in os.listdir(directory)]
+            SubTrackingAnalyzer(None, args['--stf']).combine_sub_tracking_files(files)
+        elif args['compare']:
+            if args['--cconfig'] is not None:
+                with open(args['--cconfig'], 'r') as file:
+                    config = json.load(file)
+                file_path_map = config['file_path_map'] if 'file_path_map' in config else None
+                statistic = config['statistic'] if 'statistic' in config else None
+            else:
+                file_path_map = dict[str, str]()
+                statistic = None
+            if not file_path_map and args['-m'] is None:
+                raise ValueError(
+                    f'A mapping of tracking session name to file path must be provided either through the -m option or a config file.'
+                )
+            else:
+                file_path_map.update({name: file for [name, file] in [option.split('=') for option in args['-m']]})
+            if args['--stat'] is not None:
+                statistic = args['--stat']
+            elif statistic is None:
+                statistic = 'mean'
+            comparison = TrackingComparison(file_path_map)
+            results = comparison.compare(statistic)
+            _process_output(output_format, results, output)
+    else:
+        command = args['--execute'].split()
+        if args['--tconfig'] is not None:
+            with open(args['--tconfig'], 'r') as file:
+                kwargs = json.load(file)
+        else:
+            kwargs = dict()
+        option_map = {
+            '--st': 'sleep_time',
+            '--ru': 'ram_unit',
+            '--gru': 'gpu_ram_unit',
+            '--tu': 'time_unit',
+            '--nec': 'n_expected_cores',
+            '--guuids': 'gpu_uuids',
+            '--disable-logs': 'disable_logs',
+            '--gb': 'gpu_brand',
+            '--tf': 'tracking_file',
+            '--overwrite': 'overwrite'
+        }
+        kwargs.update({
+            option_map[option]: value for option, value in args.items() if value is not None and option not in {
+                '--execute', '--output', '--format', '--help', '--version', 'sub-track', 'analyze', '--stf', 'combine', '-p', 'compare',
+                '-m', '--tconfig', '--cconfig'
+            }
+        })
+        if 'sleep_time' in kwargs.keys():
+            kwargs['sleep_time'] = float(kwargs['sleep_time'])
+        if 'n_expected_cores' in kwargs.keys():
+            kwargs['n_expected_cores'] = int(kwargs['n_expected_cores'])
+        if 'gpu_uuids' in kwargs.keys():
+            # noinspection PyUnresolvedReferences
+            kwargs['gpu_uuids'] = set(kwargs['gpu_uuids'].split(','))
+        if len(command) == 0:
+            log.error('Empty command provided.')
+            sys.exit(1)
+        try:
+            process = subp.Popen(command)
+        except FileNotFoundError:
+            log.error(f'Command not found: "{command[0]}"')
+            sys.exit(1)
+        except Exception as e:
+            log.error(f'The following error occurred when starting the command "{command[0]}":')
+            print(e)
+            sys.exit(1)
+        with Tracker(process_id=process.pid, **kwargs) as tracker:
+            process.wait()
+        print(f'Resource tracking complete. Process completed with status code: {process.returncode}')
+        _process_output(output_format, tracker, output)
+
+
+def _process_output(output_format: str, output_obj, output: str | None):
     if output_format == 'json':
-        output_str = json.dumps(tracker.to_json(), indent=1)
+        output_str = json.dumps(output_obj.to_json(), indent=1)
     elif output_format == 'text':
-        output_str = str(tracker)
+        output_str = str(output_obj)
+    elif output_format == 'pickle':
+        output_str = pkl.dumps(output_obj)
     else:
         log.error(f'"{output_format}" is not a valid format. Valid values are "json" or "text".')
         sys.exit(1)
     if output is None:
         print(output_str)
     else:
-        with open(output, 'w') as file:
+        mode = 'wb' if output_format == 'pickle' else 'w'
+        with open(output, mode) as file:
             file.write(output_str)
+
+
+if __name__ == '__main__':
+    main()