Update handler

Author: fealho

sdgym/benchmark.py

@@ -38,6 +38,7 @@
 )
 from sdmetrics.single_table import DCRBaselineProtection
 
+from sdgym import __version__ as SDGYM_VERSION
 from sdgym.datasets import get_dataset_paths, load_dataset
 from sdgym.errors import BenchmarkError, SDGymError
 from sdgym.metrics import get_metrics
@@ -1219,7 +1220,7 @@ def _write_metainfo_file(synthesizers, job_args_list, modality, result_writer=No
         'modality': modality,
         'starting_date': datetime.today().strftime('%m_%d_%Y %H:%M:%S'),
         'completed_date': None,
-        'sdgym_version': version('sdgym'),
+        'sdgym_version': SDGYM_VERSION,
         'jobs': jobs,
     }
 

sdgym/result_explorer/result_explorer.py

@@ -4,7 +4,11 @@
 
 from sdgym.benchmark import DEFAULT_SINGLE_TABLE_DATASETS
 from sdgym.datasets import load_dataset
-from sdgym.result_explorer.result_handler import LocalResultsHandler, S3ResultsHandler
+from sdgym.result_explorer.result_handler import (
+    SYNTHESIZER_BASELINE,
+    LocalResultsHandler,
+    S3ResultsHandler,
+)
 from sdgym.s3 import _get_s3_client, is_s3_path
 
 
@@ -14,6 +18,42 @@ def _validate_local_path(path):
         raise ValueError(f"The provided path '{path}' is not a valid local directory.")
 
 
+_FOLDER_BY_MODALITY = {
+    'single_table': 'single_table',
+    'multi_table': 'multi_table',
+}
+
+_BASELINE_BY_MODALITY = {
+    'multi_table': 'MultiTableUniformSynthesizer',
+}
+
+
+def _get_baseline_synthesizer(modality):
+    """Return the appropriate baseline synthesizer for the given modality."""
+    return _BASELINE_BY_MODALITY.get(modality, SYNTHESIZER_BASELINE)
+
+
+def _resolve_effective_path(path, modality):
+    """Append the modality folder to the given base path if provided."""
+    if not modality:
+        return path
+
+    folder = _FOLDER_BY_MODALITY.get(modality)
+    if folder is None:
+        valid = ', '.join(sorted(_FOLDER_BY_MODALITY))
+        raise ValueError(f'Invalid modality "{modality}". Valid options are: {valid}.')
+
+    # Avoid double-appending if already included
+    if str(path).rstrip('/').endswith(('/' + folder, folder)):
+        return path
+
+    if is_s3_path(path):
+        path = path.rstrip('/') + '/' + folder
+        return path
+
+    return os.path.join(path, folder)
+
+
 class ResultsExplorer:
     """Explorer for SDGym benchmark results, supporting both local and S3 storage."""
 
@@ -22,12 +62,18 @@ def __init__(self, path, aws_access_key_id=None, aws_secret_access_key=None):
         self.aws_access_key_id = aws_access_key_id
         self.aws_secret_access_key = aws_secret_access_key
 
+        baseline_synthesizer = _get_baseline_synthesizer(modality)
+        effective_path = _resolve_effective_path(path, modality)
         if is_s3_path(path):
             s3_client = _get_s3_client(path, aws_access_key_id, aws_secret_access_key)
-            self._handler = S3ResultsHandler(path, s3_client)
+            self._handler = S3ResultsHandler(
+                effective_path, s3_client, baseline_synthesizer=baseline_synthesizer
+            )
         else:
-            _validate_local_path(path)
-            self._handler = LocalResultsHandler(path)
+            _validate_local_path(effective_path)
+            self._handler = LocalResultsHandler(
+                effective_path, baseline_synthesizer=baseline_synthesizer
+            )
 
     def list(self):
         """List all runs available in the results directory."""

sdgym/result_explorer/result_handler.py

@@ -22,6 +22,10 @@
 class ResultsHandler(ABC):
     """Abstract base class for handling results storage and retrieval."""
 
+    def __init__(self, baseline_synthesizer=SYNTHESIZER_BASELINE):
+        # Allow overrides per modality while maintaining the historical default.
+        self.baseline_synthesizer = baseline_synthesizer or SYNTHESIZER_BASELINE
+
     @abstractmethod
     def list(self):
         """List all runs in the results directory."""
@@ -59,7 +63,8 @@ def _compute_wins(self, result):
         result['Win'] = 0
         for dataset in datasets:
             score_baseline = result.loc[
-                (result['Synthesizer'] == SYNTHESIZER_BASELINE) & (result['Dataset'] == dataset)
+                (result['Synthesizer'] == self.baseline_synthesizer)
+                & (result['Dataset'] == dataset)
             ]['Quality_Score'].to_numpy()
             if score_baseline.size == 0:
                 continue
@@ -84,7 +89,7 @@ def _get_summarize_table(self, folder_to_results, folder_infos):
                 f' - # datasets: {folder_infos[folder]["# datasets"]}'
                 f' - sdgym version: {folder_infos[folder]["sdgym_version"]}'
             )
-            results = results.loc[results['Synthesizer'] != SYNTHESIZER_BASELINE]
+            results = results.loc[results['Synthesizer'] != self.baseline_synthesizer]
             column_data = results.groupby(['Synthesizer'])['Win'].sum()
             columns.append((date_obj, column_name, column_data))
 
@@ -107,9 +112,11 @@ def _get_column_name_infos(self, folder_to_results):
                 continue
 
             metainfo_info = self._load_yaml_file(folder, yaml_files[0])
-            num_datasets = results.loc[
-                results['Synthesizer'] == SYNTHESIZER_BASELINE, 'Dataset'
-            ].nunique()
+            baseline_mask = results['Synthesizer'] == self.baseline_synthesizer
+            if baseline_mask.any():
+                num_datasets = results.loc[baseline_mask, 'Dataset'].nunique()
+            else:
+                num_datasets = results['Dataset'].nunique()
             folder_to_info[folder] = {
                 'date': metainfo_info.get('starting_date')[:NUM_DIGITS_DATE],
                 'sdgym_version': metainfo_info.get('sdgym_version'),
@@ -236,7 +243,8 @@ def all_runs_complete(self, folder_name):
 class LocalResultsHandler(ResultsHandler):
     """Results handler for local filesystem."""
 
-    def __init__(self, base_path):
+    def __init__(self, base_path, baseline_synthesizer=SYNTHESIZER_BASELINE):
+        super().__init__(baseline_synthesizer=baseline_synthesizer)
         self.base_path = base_path
 
     def list(self):
@@ -287,7 +295,8 @@ def _load_yaml_file(self, folder_name, file_name):
 class S3ResultsHandler(ResultsHandler):
     """Results handler for AWS S3 storage."""
 
-    def __init__(self, path, s3_client):
+    def __init__(self, path, s3_client, baseline_synthesizer=SYNTHESIZER_BASELINE):
+        super().__init__(baseline_synthesizer=baseline_synthesizer)
         self.s3_client = s3_client
         self.bucket_name = path.split('/')[2]
         self.prefix = '/'.join(path.split('/')[3:]).rstrip('/') + '/'
@@ -396,8 +405,8 @@ def _get_results(self, folder_name, file_names):
         for file_name in file_names:
             s3_key = f'{self.prefix}{folder_name}/{file_name}'
             response = self.s3_client.get_object(Bucket=self.bucket_name, Key=s3_key)
-            df = pd.read_csv(io.BytesIO(response['Body'].read()))
-            results.append(df)
+            result_df = pd.read_csv(io.BytesIO(response['Body'].read()))
+            results.append(result_df)
 
         return results
 

tests/integration/result_explorer/_benchmark_results/multi_table/SDGym_results_12_02_2025/fake_hotels_12_02_2025/HMASynthesizer/HMASynthesizer.pkl

@@ -0,0 +1,2 @@
+Synthesizer,Dataset,Dataset_Size_MB,Train_Time,Peak_Memory_MB,Synthesizer_Size_MB,Sample_Time,Evaluate_Time,Diagnostic_Score,Quality_Score
+HMASynthesizer,fake_hotels,0.048698,22.852492,33.315142,0.988611,2.723049,0.082362,1.0,0.7353482911012336

tests/integration/result_explorer/_benchmark_results/multi_table/SDGym_results_12_02_2025/fake_hotels_12_02_2025/HMASynthesizer/HMASynthesizer_benchmark_result.csv

@@ -0,0 +1,2 @@
+Synthesizer,Dataset,Dataset_Size_MB,Train_Time,Peak_Memory_MB,Synthesizer_Size_MB,Sample_Time,Evaluate_Time,Diagnostic_Score,Quality_Score
+MultiTableUniformSynthesizer,fake_hotels,0.048698,0.201284,0.851853,0.109464,0.02749,0.081629,0.9122678149273894,0.5962941240006595

tests/integration/result_explorer/_benchmark_results/multi_table/SDGym_results_12_02_2025/fake_hotels_12_02_2025/HMASynthesizer/HMASynthesizer_synthetic_data.zip

@@ -0,0 +1,11 @@
+completed_date: 12_02_2025 08:28:47
+jobs:
+- - fake_hotels
+  - MultiTableUniformSynthesizer
+- - fake_hotels
+  - HMASynthesizer
+modality: multi_table
+run_id: run_12_02_2025_0
+sdgym_version: 0.11.2.dev0
+sdv_version: 1.28.0
+starting_date: 12_02_2025 08:28:21