Completes the analysis test

Author: erikhuck

src/gpu_tracker/_helper_classes.py

@@ -13,6 +13,10 @@
 
 _SUMMARY_STATS = ['min', 'max', 'mean', 'std']
 
+def _summary_stats(data: pd.Series | pd.DataFrame) -> pd.Series | pd.DataFrame:
+    stats = data.describe().loc[_SUMMARY_STATS]
+    stats.loc['std'] = data.std(ddof=0)
+    return stats
 
 class _GPUQuerier(abc.ABC):
     command = None
@@ -348,7 +352,8 @@ def load_code_block_names(self) -> list[str]:
         return sorted(self.timestamps.code_block_name.unique())
 
     def _overall_timepoint_results(self, fields: list[str]) -> pd.DataFrame:
-        return self.timepoints[fields].describe().loc[_SUMMARY_STATS].T
+        stats = _summary_stats(self.timepoints[fields])
+        return stats.T
 
 
 class _SQLiteDataProxy(_DataProxy):
@@ -441,21 +446,27 @@ def load_timepoints(self, timestamp_pairs: list[tuple[float, float]]) -> pd.Data
         return self._read_sql(sql)
 
     def _overall_timepoint_results(self, fields: list[str]) -> pd.DataFrame:
-        sql = 'SELECT\n'
-        std_func = 'sqrt((sum({0} * {0}) - (sum({0}) * sum({0})) / count({0})) / count({0})) AS "STDDEV({0})"'
-        sql_funcs = 'MIN', 'MAX', 'AVG', 'STDDEV'
-        field_aggregates = list[str]()
-        for func in sql_funcs:
-            for field in fields:
-                aggregate = f'{func}({field})' if func != 'STDDEV' else std_func.format(field)
-                field_aggregates.append(aggregate)
-        sql += ',\n'.join(field_aggregates)
-        sql += f'\nFROM {_SQLiteDataProxy._DATA_TABLE}'
+        cte_blocks = list[str]()
+        selects = list[str]()
+        for col in fields:
+            mean_cte = f'mean_{col}'
+            diff_cte = f'diff_{col}'
+            cte_blocks.append(f'{mean_cte} AS (SELECT AVG({col}) AS mean FROM {_SQLiteDataProxy._DATA_TABLE})')
+            cte_blocks.append(
+                f'{diff_cte} AS (SELECT {col} - (SELECT mean FROM {mean_cte}) AS diff FROM {_SQLiteDataProxy._DATA_TABLE})'
+            )
+            selects.append(f'MIN({col})')
+            selects.append(f'MAX({col})')
+            selects.append(f'(SELECT mean FROM {mean_cte}) AS "AVG({col})"')
+            selects.append(f'(SELECT SQRT(AVG(diff * diff)) FROM {diff_cte}) AS "STDDEV({col})"')
+        with_clause = "WITH " + ",\n     ".join(cte_blocks)
+        select_clause = "SELECT " + ",\n       ".join(selects)
+        sql = f"{with_clause}\n{select_clause} FROM {_SQLiteDataProxy._DATA_TABLE};"
         results = self._read_sql(sql).squeeze()
         reshaped_results = pd.DataFrame()
-        n_fields = len(fields)
-        for i, sql_func, index in zip(range(0, len(results), n_fields), sql_funcs, _SUMMARY_STATS):
-            next_row = results.iloc[i: i + n_fields]
+        sql_funcs = 'MIN', 'MAX', 'AVG', 'STDDEV'
+        for sql_func, index in zip(sql_funcs, _SUMMARY_STATS):
+            next_row = results.loc[[idx.startswith(sql_func) for idx in results.index]]
             next_row.index = [col.replace(sql_func, '').replace('(', '').replace(')', '') for col in next_row.index]
             reshaped_results.loc[:, index] = next_row
         return reshaped_results

src/gpu_tracker/sub_tracker.py

@@ -9,7 +9,7 @@
 import pickle as pkl
 import logging as log
 import typing as typ
-from ._helper_classes import _DataProxy, _SubTrackerLog, _SUMMARY_STATS
+from ._helper_classes import _DataProxy, _SubTrackerLog, _SUMMARY_STATS, _summary_stats
 
 
 class SubTracker:
@@ -168,7 +168,7 @@ def sub_tracking_results(self) -> SubTrackingResults:
         for code_block_name in code_block_names:
             time_stamp_pairs = self.load_timestamp_pairs(code_block_name)
             time_stamp_diffs = pd.Series([stop_time - start_time for (start_time, stop_time) in time_stamp_pairs])
-            compute_time_results = time_stamp_diffs.describe()[_SUMMARY_STATS]
+            compute_time_results = _summary_stats(time_stamp_diffs)
             compute_time_results['total'] = time_stamp_diffs.sum().item()
             timepoints = self.load_timepoints(time_stamp_pairs)
             num_non_empty_calls = sum(
@@ -179,11 +179,11 @@ def sub_tracking_results(self) -> SubTrackingResults:
                 ]
             )
             timepoints = timepoints.drop(columns='timestamp')
+            resource_usage = _summary_stats(timepoints).T
             code_block_results.append(
                 CodeBlockResults(
                     name=code_block_name, num_timepoints=len(timepoints), num_calls=len(time_stamp_pairs),
-                    num_non_empty_calls=num_non_empty_calls, compute_time=compute_time_results,
-                    resource_usage=timepoints.describe().loc[_SUMMARY_STATS].T
+                    num_non_empty_calls=num_non_empty_calls, compute_time=compute_time_results, resource_usage=resource_usage
                 )
             )
         return SubTrackingResults(overall_results, static_data, code_block_results)
@@ -409,7 +409,10 @@ def _dict_to_str(string: str, results: dict, indent: int, no_title_keys: set[str
             string = _dict_to_str(string, value, indent + 1, no_title_keys)
         elif type(value) is pd.DataFrame:
             string += f'{key}:\n'
-            with pd.option_context('display.max_rows', None, 'display.max_columns', None, 'display.width', 5000):
+            with pd.option_context(
+                    'display.max_rows', None, 'display.max_columns', None, 'display.width', 5000, 'display.float_format',
+                    lambda x: f'{float(f"{x:.8f}")}'
+            ):
                 df_str = str(value)
             df_str = '\n'.join(indent_str + '\t' + line for line in df_str.splitlines())
             string += df_str + '\n'
@@ -418,7 +421,7 @@ def _dict_to_str(string: str, results: dict, indent: int, no_title_keys: set[str
             for value in value:
                 string = _dict_to_str(string, value, indent + 1, no_title_keys) + '\n'
         else:
-            value = f'{value:.4f}' if type(value) is float else value
+            value = f'{value:.8f}' if type(value) is float else value
             string += f'{key}:{" " * (max_key_len - len(key))} {value}\n'
     return string
 

src/gpu_tracker/tracker.py

@@ -65,6 +65,7 @@ def __init__(
         self._is_linux = platform.system().lower() == 'linux'
         cannot_connect_warning = ('The {} command is installed but cannot connect to a GPU. '
                                   'The GPU RAM and GPU utilization values will remain 0.0.')
+        self.data_proxy = _DataProxy.create(tracking_file, overwrite)
         if gpu_brand is None:
             nvidia_available = _NvidiaQuerier.is_available()
             nvidia_installed = nvidia_available is not None
@@ -119,7 +120,6 @@ def __init__(
         self._resource_usage = ResourceUsage(
             max_ram=max_ram, max_gpu_ram=max_gpu_ram, cpu_utilization=cpu_utilization, gpu_utilization=gpu_utilization,
             compute_time=compute_time)
-        self.data_proxy = _DataProxy.create(tracking_file, overwrite)
         if self.data_proxy is not None:
             static_data = _StaticData(
                 ram_unit, gpu_ram_unit, time_unit, max_ram.system_capacity, max_gpu_ram.system_capacity, system_core_count,

tests/data/decorated-function-other-file.csv

@@ -1,3 +1,3 @@
 position,process_id,timestamp
-START,1234,12
-STOP,1234,13
+0,1234,12
+1,1234,13

tests/data/decorated-function.csv

@@ -1,13 +1,13 @@
 position,process_id,timestamp
-START,1234,0
-STOP,1234,1
-START,1234,2
-STOP,1234,3
-START,1234,4
-STOP,1234,5
-START,1234,6
-STOP,1234,7
-START,1234,8
-STOP,1234,9
-START,1234,10
-STOP,1234,11
+0,1234,0
+1,1234,1
+0,1234,2
+1,1234,3
+0,1234,4
+1,1234,5
+0,1234,6
+1,1234,7
+0,1234,8
+1,1234,9
+0,1234,10
+1,1234,11

tests/data/sub-tracker.csv

@@ -1,11 +1,11 @@
 position,process_id,timestamp
-START,1234,0
-STOP,1234,1
-START,1234,2
-STOP,1234,3
-START,1234,4
-STOP,1234,5
-START,1234,6
-STOP,1234,7
-START,1234,8
-STOP,1234,9
+0,1234,0
+1,1234,1
+0,1234,2
+1,1234,3
+0,1234,4
+1,1234,5
+0,1234,6
+1,1234,7
+0,1234,8
+1,1234,9