rename 'error' column to 'Error'

Author: R-Palazzo

sdgym/benchmark.py

@@ -514,7 +514,7 @@ def _compute_scores(
         for metric_name, metric in metrics.items():
             scores.append({
                 'metric': metric_name,
-                'error': 'Metric Timeout',
+                'Error': 'Metric Timeout',
             })
             # re-inject list to multiprocessing output
             output['scores'] = scores
@@ -537,7 +537,7 @@ def _compute_scores(
             scores[-1].update({
                 'score': score,
                 'normalized_score': normalized_score,
-                'error': error,
+                'Error': error,
                 'metric_time': calculate_score_time(start),
             })
             # re-inject list to multiprocessing output
@@ -603,7 +603,7 @@ def _score(
         output = {}
 
     output['timeout'] = True  # To be deleted if there is no error
-    output['error'] = 'Load Timeout'  # To be deleted if there is no error
+    output['Error'] = 'Load Timeout'  # To be deleted if there is no error
     try:
         LOGGER.info(
             'Running %s on %s dataset %s; %s',
@@ -615,7 +615,7 @@ def _score(
 
         output['dataset_size'] = get_size_of(data) / N_BYTES_IN_MB
         # To be deleted if there is no error
-        output['error'] = 'Synthesizer Timeout'
+        output['Error'] = 'Synthesizer Timeout'
 
         try:
             synthetic_data, train_time, sample_time, synthesizer_size, peak_memory = _synthesize(
@@ -642,7 +642,7 @@ def _score(
             )
 
             # No error so far. _compute_scores tracks its own errors by metric
-            del output['error']
+            del output['Error']
             _compute_scores(
                 metrics,
                 data,
@@ -671,14 +671,14 @@ def _score(
             output['peak_memory'] = err.peak_memory
 
             output['exception'] = err.exception
-            output['error'] = err.error
+            output['Error'] = err.error
             output['timeout'] = False
 
     except Exception:
         LOGGER.exception('Error running %s on dataset %s;', synthesizer['name'], dataset_name)
         exception, error = format_exception()
         output['exception'] = exception
-        output['error'] = error
+        output['Error'] = error
         output['timeout'] = False  # There was no timeout
 
     finally:
@@ -744,7 +744,7 @@ def _score_with_timeout(
         thread.join(timeout)
         if thread.is_alive():
             LOGGER.error('Timeout running %s on dataset %s;', synthesizer['name'], dataset_name)
-            return {'timeout': True, 'error': 'Synthesizer Timeout'}
+            return {'timeout': True, 'Error': 'Synthesizer Timeout'}
 
         return output
 
@@ -815,8 +815,8 @@ def _format_output(
     for score in output.get('scores', []):
         scores.insert(len(scores.columns), score['metric'], score['normalized_score'])
 
-    if 'error' in output:
-        scores['error'] = output['error']
+    if 'Error' in output:
+        scores['Error'] = output['Error']
 
     return scores
 
@@ -1085,8 +1085,8 @@ def _add_adjusted_scores(scores, timeout):
 
         fit_times = scores.loc[dataset_mask, 'Train_Time'].fillna(0)
         sample_times = scores.loc[dataset_mask, 'Sample_Time'].fillna(0)
-        if 'error' in scores.columns:
-            errors = scores.loc[dataset_mask, 'error']
+        if 'Error' in scores.columns:
+            errors = scores.loc[dataset_mask, 'Error']
         else:
             errors = pd.Series([None] * dataset_mask.sum(), index=scores.index[dataset_mask])
 

sdgym/cli/__main__.py

@@ -39,13 +39,13 @@ def _print_table(data, sort=None, reverse=False, format=None):
         for field, formatter in format.items():
             data[field] = data[field].apply(formatter)
 
-    if 'error' in data:
-        error = data['error']
+    if 'Error' in data:
+        error = data['Error']
         if pd.isna(error).all():
-            del data['error']
+            del data['Error']
         else:
             long_error = error.str.len() > 30
-            data.loc[long_error, 'error'] = error[long_error].str[:30] + '...'
+            data.loc[long_error, 'Error'] = error[long_error].str[:30] + '...'
 
     print(tabulate.tabulate(data, tablefmt='github', headers=data.columns, showindex=False))  # noqa: T201
 

sdgym/cli/summary.py

@@ -35,11 +35,11 @@ def preprocess(data):
     bydataset = grouped.mean()
     data = bydataset.reset_index()
 
-    if 'error' in data.columns:
+    if 'Error' in data.columns:
         errors = data.error.fillna('')
         for message, column in KNOWN_ERRORS:
             data[column] = errors.str.contains(message)
-            data.loc[data[column], 'error'] = np.nan
+            data.loc[data[column], 'Error'] = np.nan
 
     return data
 
@@ -122,7 +122,7 @@ def summarize(data, baselines=(), datasets=None):
         baseline_scores = baseline_data.set_index('Dataset').Quality_Score
         results[f'beat_{baseline.lower()}'] = _beat_baseline(data, baseline_scores)
 
-    if 'error' in data.columns:
+    if 'Error' in data.columns:
         grouped = data.groupby('Synthesizer')
         for _, error_column in KNOWN_ERRORS:
             results[error_column] = grouped[error_column].sum()
@@ -135,7 +135,7 @@ def summarize(data, baselines=(), datasets=None):
 
 
 def _error_counts(data):
-    if 'error' in data.columns:
+    if 'Error' in data.columns:
         return data.error.value_counts()
     return 0
 
@@ -158,8 +158,8 @@ def errors_summary(data):
     Returns:
         pandas.DataFrame
     """
-    if 'error' in data.columns:
-        all_errors = pd.DataFrame(_error_counts(data)).rename(columns={'error': 'all'})
+    if 'Error' in data.columns:
+        all_errors = pd.DataFrame(_error_counts(data)).rename(columns={'Error': 'all'})
         synthesizer_errors = data.groupby('Synthesizer').apply(_error_counts).pivot_table(level=0)
         for synthesizer, errors in synthesizer_errors.items():
             all_errors[synthesizer] = errors.fillna(0).astype(int)

sdgym/run_benchmark/upload_benchmark_results.py

@@ -231,8 +231,6 @@ def get_model_details(summary, results, df_to_plot, modality):
     with open(SYNTHESIZER_DESCRIPTION_PATH, 'r', encoding='utf-8') as f:
         synthesizer_info = yaml.safe_load(f) or {}
 
-    err_column = next((c for c in ('error', 'Error') if c in results.columns), None)
-    err_column_flag = err_column is not None
     paretos_synthesizers = (
         df_to_plot.loc[df_to_plot['Pareto'].eq(True), 'Synthesizer'].astype(str).add('Synthesizer')
     )
@@ -259,16 +257,16 @@ def get_model_details(summary, results, df_to_plot, modality):
     model_details['Number of datasets - Wins'] = (
         model_details['Synthesizer'].map(wins).fillna(0).astype(int)
     )
-    if err_column_flag:
+    if 'Error' in results.columns:
         timeout_counts = (
             results
-            .loc[results[err_column].eq('Synthesizer Timeout')]
+            .loc[results['Error'].eq('Synthesizer Timeout')]
             .groupby('Synthesizer')['Dataset']
             .nunique()
         )
         error_counts = (
             results
-            .loc[results[err_column].notna() & ~results[err_column].eq('Synthesizer Timeout')]
+            .loc[results['Error'].notna() & ~results['Error'].eq('Synthesizer Timeout')]
             .groupby('Synthesizer')['Dataset']
             .nunique()
         )

tests/integration/test_benchmark.py

@@ -142,7 +142,7 @@ def test_benchmark_single_table_error_handling():
     assert not output.empty
     assert 'Train_Time' in output
     assert 'Sample_Time' in output
-    assert output['error'].to_list() == [np.nan, np.nan, np.nan, 'ValueError: random error']
+    assert output['Error'].to_list() == [np.nan, np.nan, np.nan, 'ValueError: random error']
 
 
 def test_benchmark_single_table_compute_quality_score():
@@ -318,7 +318,7 @@ def test_benchmark_single_table_timeout():
             'Diagnostic_Score': None,
             'Quality_Score': None,
             'Privacy_Score': None,
-            'error': 'Synthesizer Timeout',
+            'Error': 'Synthesizer Timeout',
             'Adjusted_Total_Time': 1 + fallback_train_time + fallback_sample_time,
             'Adjusted_Quality_Score': None,
         },
@@ -508,7 +508,7 @@ def test_benchmark_single_table_no_synthesizers_with_parameters():
         .all()
     )
     assert result['Evaluate_Time'] is None
-    assert result['error'] == 'ValueError: Unknown single_table metric: a'
+    assert result['Error'] == 'ValueError: Unknown single_table metric: a'
 
 
 def test_benchmark_single_table_custom_synthesizer():
@@ -811,7 +811,7 @@ def fit(self, data):
     )
 
     # Assert
-    assert result['error'].to_list() == [
+    assert result['Error'].to_list() == [
         'Exception: Fitting error',
         np.nan,
         np.nan,
@@ -855,7 +855,7 @@ def sample(self, num_rows):
     )
 
     # Assert
-    assert result['error'].to_list() == [
+    assert result['Error'].to_list() == [
         'Exception: Sampling error',
         np.nan,
         np.nan,
@@ -1034,7 +1034,7 @@ def _augment_tables(self, data):
     )
 
     # Assert
-    assert result['error'].to_list() == [
+    assert result['Error'].to_list() == [
         'Exception: Fitting error',
         np.nan,
         'Exception: Fitting error',

tests/unit/run_benchmark/test_upload_benchmark_result.py

@@ -308,7 +308,7 @@ def test_get_model_details(mock_open, mock_yaml_load):
             'CTGANSynthesizer',
         ],
         'Quality_Score': [0.1, 0.2, 0.3, 0.15, 0.25],
-        'error': [
+        'Error': [
             'Synthesizer Timeout',  # timeout on D1 for GaussianCopulaSynthesizer
             'Other Error',  # error on D2 for GaussianCopulaSynthesizer
             None,  # no error on D3 for GaussianCopulaSynthesizer

tests/unit/test_benchmark.py

@@ -235,7 +235,7 @@ def test_benchmark_single_table_with_timeout(mock_multiprocessing, mock__score):
     # Setup
     mocked_process = mock_multiprocessing.Process.return_value
     manager = mock_multiprocessing.Manager.return_value
-    manager_dict = {'timeout': True, 'error': 'Synthesizer Timeout'}
+    manager_dict = {'timeout': True, 'Error': 'Synthesizer Timeout'}
     manager.__enter__.return_value.dict.return_value = manager_dict
 
     # Run
@@ -261,7 +261,7 @@ def test_benchmark_single_table_with_timeout(mock_multiprocessing, mock__score):
         'Diagnostic_Score': {0: None},
         'Quality_Score': {0: None},
         'Privacy_Score': {0: None},
-        'error': {0: 'Synthesizer Timeout'},
+        'Error': {0: 'Synthesizer Timeout'},
         'Adjusted_Total_Time': {0: None},
         'Adjusted_Quality_Score': {0: None},
     })
@@ -357,14 +357,14 @@ def test__format_output():
         'scores': [
             {
                 'metric': 'NewRowSynthesis',
-                'error': None,
+                'Error': None,
                 'score': 0.998,
                 'normalized_score': 0.998,
                 'metric_time': 6.0,
             },
             {
                 'metric': 'NewMetric',
-                'error': None,
+                'Error': None,
                 'score': 0.998,
                 'normalized_score': 0.998,
                 'metric_time': 5.0,
@@ -985,15 +985,15 @@ def test__add_adjusted_scores_timeout():
         'Train_Time': [np.nan, 0.5],
         'Sample_Time': [np.nan, 0.25],
         'Quality_Score': [np.nan, 0.5],
-        'error': ['Synthesizer Timeout', np.nan],
+        'Error': ['Synthesizer Timeout', np.nan],
     })
     expected = pd.DataFrame({
         'Synthesizer': ['GaussianCopulaSynthesizer', 'UniformSynthesizer'],
         'Dataset': ['dataset1', 'dataset1'],
         'Train_Time': [np.nan, 0.5],
         'Sample_Time': [np.nan, 0.25],
         'Quality_Score': [np.nan, 0.5],
-        'error': ['Synthesizer Timeout', np.nan],
+        'Error': ['Synthesizer Timeout', np.nan],
         'Adjusted_Total_Time': [10.75, 1.25],
         'Adjusted_Quality_Score': [0.5, 0.5],
     })
@@ -1014,15 +1014,15 @@ def test__add_adjusted_scores_errors():
         'Train_Time': [np.nan, 1.0, 1.0, 0.5],
         'Sample_Time': [np.nan, np.nan, 2.0, 0.25],
         'Quality_Score': [np.nan, np.nan, np.nan, 0.5],
-        'error': ['ValueError', 'RuntimeError', 'KeyError', np.nan],
+        'Error': ['ValueError', 'RuntimeError', 'KeyError', np.nan],
     })
     expected = pd.DataFrame({
         'Synthesizer': ['ErrorOnTrain', 'ErrorOnSample', 'ErrorAfterSample', 'UniformSynthesizer'],
         'Dataset': ['dataset1', 'dataset1', 'dataset1', 'dataset1'],
         'Train_Time': [np.nan, 1.0, 1.0, 0.5],
         'Sample_Time': [np.nan, np.nan, 2.0, 0.25],
         'Quality_Score': [np.nan, np.nan, np.nan, 0.5],
-        'error': ['ValueError', 'RuntimeError', 'KeyError', np.nan],
+        'Error': ['ValueError', 'RuntimeError', 'KeyError', np.nan],
         'Adjusted_Total_Time': [0.75, 1.75, 3.75, 1.25],
         'Adjusted_Quality_Score': [0.5, 0.5, 0.5, 0.5],
     })

tests/unit/test_summary.py

@@ -64,7 +64,7 @@ def test_make_summary_spreadsheet(
         index=['synth1', 'synth2'],
     )
     preprocessed_data = pd.DataFrame({'modality': ['single-table']})
-    errors = pd.DataFrame({'synth1': [0], 'synth2': [1], 'error': ['RuntimeError: error.']})
+    errors = pd.DataFrame({'synth1': [0], 'synth2': [1], 'Error': ['RuntimeError: error.']})
     preprocess_mock.return_value = preprocessed_data
     summarize_mock.return_value = data
     errors_summary_mock.return_value = errors